def test_Define_Final_Catchment():
"""test function that will:
Generate the final lake river routing structure by merging subbasin
polygons that are covered by the same lake.
The input are the catchment polygons and river segements
before merging for lakes. The input files can be output of
any of following functions:
SelectLakes, Select_Routing_product_based_SubId,
Customize_Routing_Topology,RoutingNetworkTopologyUpdateToolset_riv
The result is the final catchment polygon that ready to be used for
hydrological modeling
"""
###Floder where store the inputs for tests function
Routing_Product_Folder = "./testdata/Simplified_By_DA/"
###Folder where store the expected resuts
Expect_Result_Folder = os.path.join("./testdata", "Simplified_By_DA")
###Folder where the output will be generated
Output_Folder = os.path.join("./testdata", "testout10")
###The pathes for all inputs
Path_final_riv_ply = os.path.join(
Routing_Product_Folder, "finalriv_info_ply.shp") ### River polyline
Path_final_riv = os.path.join(Routing_Product_Folder,
"finalriv_info.shp") ### Catchment polygons
###Generate test resuts
RTtool = LRRT()
RTtool.Define_Final_Catchment(
Path_final_rivply=Path_final_riv_ply,
Path_final_riv=Path_final_riv,
OutputFolder=Output_Folder,
)
"""Evaluate attribute table of two polygons
"""
### transfer expected product into pandas dataframe
Expect_Finalcat_info = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder,
"finalcat_info.shp")).sort_values(by=["SubId"])
### transfer resulted product into pandas dataframe
Result_Finalcat_info = Dbf_To_Dataframe(
os.path.join(Output_Folder,
"finalcat_info.shp")).sort_values(by=["SubId"])
assert Evaluate_Two_Dataframes(Expect_Finalcat_info,
Result_Finalcat_info,
Check_Col_NM="SubId")
shutil.rmtree(Output_Folder)
def test_RoutingNetworkTopologyUpdateToolset_riv():
"""test function that will:
Calculate hydrological paramters for each subbasin generated by
"AutomatedWatershedsandLakesFilterToolset". The result generaed
by this tool can be used as inputs for Define_Final_Catchment
and other post processing tools
"""
###Floder where store the inputs for tests function
Data_Folder = "./testdata/Required_data_to_start_from_dem/"
###Folder where store the expected resuts
Final_Result_Folder_Expected = os.path.join("./testdata",
"Final_output_folder",
"Expected_InDEM")
Temporary_Result_Folder_Expected = os.path.join("./testdata",
"Temporary_output_folder",
"Expected_InDEM")
###Folder where the output will be generated
Temporary_Result_Folder_Result = os.path.join("./testdata",
"Temporary_output_folder",
"testout4")
Final_Result_Folder_Result = os.path.join("./testdata",
"Final_output_folder",
"testout4")
# shutil.rmtree(Temporary_Result_Folder_Result,ignore_errors=True)
###The pathes for all inputs
Path_DEM_big = os.path.join(Data_Folder, "DEM_big_merit.tif")
Path_DEM_small = os.path.join(Data_Folder, "DEM_samll_merit.tif")
Path_Lake_ply = os.path.join(Data_Folder, "HyLake.shp")
Path_bkf_wd = os.path.join(Data_Folder, "Bkfullwidth_depth.shp")
Path_Landuse = os.path.join(Data_Folder, "landuse.tif")
Path_Roughness_landuse = os.path.join(Data_Folder, "Landuse_info.csv")
###Generate test resuts, for option 1
RTtool = LRRT(
dem_in=Path_DEM_small,
WidDep=Path_bkf_wd,
Lakefile=Path_Lake_ply,
Landuse=Path_Landuse,
Landuseinfo=Path_Roughness_landuse,
OutputFolder=Final_Result_Folder_Result,
TempOutFolder=Temporary_Result_Folder_Result,
)
### test using extent of input dem as processing extent
RTtool.Generatmaskregion()
RTtool.Generateinputdata()
RTtool.WatershedDiscretizationToolset(accthresold=500)
RTtool.AutomatedWatershedsandLakesFilterToolset(
Thre_Lake_Area_Connect=0, Thre_Lake_Area_nonConnect=0)
RTtool.RoutingNetworkTopologyUpdateToolset_riv(projection="EPSG:3573")
"""Evaluate result attribute table of result polygon,polylie, and two
lake polygons
"""
### transfer expected siplified product into pandas dataframe
Expect_Finalriv_info_ply = Dbf_To_Dataframe(
os.path.join(Final_Result_Folder_Expected,
"finalriv_info_ply.shp")).sort_values(by=["SubId"])
Result_Finalriv_info_ply = Dbf_To_Dataframe(
os.path.join(Final_Result_Folder_Result,
"finalriv_info_ply.shp")).sort_values(by=["SubId"])
assert Evaluate_Two_Dataframes(Result_Finalriv_info_ply,
Expect_Finalriv_info_ply,
Check_Col_NM="SubId")
"""Evaluate lake polygon files
Con_Lake_Ply is the lake polygon that connected by river network
Non_Con_Lake_Ply is the lake polygon that did not connected by
river network
"""
### transfer expected siplified connected lake polygon into pandas dataframe Expect_Con_Lake_Ply
Expect_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder,
"Con_Lake_Ply.shp")).sort_values(by=["Hylak_id"])
### transfer expected siplified non connected lake polygon into pandas dataframe Expect_Non_Con_Lake_Ply
Expect_Non_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder,
"Non_Con_Lake_Ply.shp")).sort_values(by=["Hylak_id"])
### transfer resulted siplified connected lake polygon into pandas dataframe Result_Con_Lake_Ply
Result_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Output_Folder,
"Con_Lake_Ply.shp")).sort_values(by=["Hylak_id"])
### transfer resulted siplified non connected lake polygon into pandas dataframe Result_Non_Con_Lake_Ply
Result_Non_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Output_Folder,
"Non_Con_Lake_Ply.shp")).sort_values(by=["Hylak_id"])
### compare two pandas dataframe Expect_Con_Lake_Ply and Result_Con_Lake_Ply
assert Evaluate_Two_Dataframes(Expect_Con_Lake_Ply,
Result_Con_Lake_Ply,
Check_Col_NM="Hylak_id")
### compare two pandas dataframe Expect_Non_Con_Lake_Ply and Result_Non_Con_Lake_Ply
assert Evaluate_Two_Dataframes(Expect_Non_Con_Lake_Ply,
Result_Non_Con_Lake_Ply,
Check_Col_NM="Hylak_id")
RTtool.Output_Clean()
def test_GenerateHRUS():
"""test function that will:
be used to overlay: subbasin polygon, lake polygon (optional)
, Land use polygon (optional), soil type polygon(optional),
vegetation polygon (optional), and two other user defined polygons
(optional).
Non-lake HRU polygons in a subbasin is defined by an unique
combination of all user provided datasets.
A lake HRU polygon is defined the same as the provided lake polygon.
All value of landuse and Veg polygon covered by lake will
be changed to 1, indicating it is a covered by lake.
All value of the soil polygon covered by the lake will be change to
the soil id of the polygon covered by the lake with largest area.
"""
###Floder where store the inputs for tests function
Routing_Product_Folder = "./testdata/Simplified_By_DA/"
HRU_Folder = os.path.join("./testdata", "HRU")
###Folder where store the expected resuts
Expect_Result_Folder = os.path.join("./testdata", "Simplified_By_DA")
###Folder where the output will be generated
Output_Folder = os.path.join("./testdata", "testout10")
###Generate test resuts
RTtool = LRRT()
RTtool.GenerateHRUS(
OutputFolder=Output_Folder,
Path_Subbasin_Ply=os.path.join(Routing_Product_Folder,
"finalcat_info.shp"),
Path_Connect_Lake_ply=os.path.join(Routing_Product_Folder,
"Con_Lake_Ply.shp"),
Path_Non_Connect_Lake_ply=os.path.join(Routing_Product_Folder,
"Non_Con_Lake_Ply.shp"),
Path_Landuse_Ply="#",
Landuse_ID="Landuse_ID",
Path_Soil_Ply="#",
Soil_ID="Soil_ID",
Path_Veg_Ply="#",
Veg_ID="Veg_ID",
Path_Other_Ply_1="#",
Other_Ply_ID_1="O_ID_1",
Path_Other_Ply_2="#",
Other_Ply_ID_2="O_ID_2",
Landuse_info=os.path.join(HRU_Folder, "landuse_info.csv"),
Soil_info=os.path.join(HRU_Folder, "soil_info.csv"),
Veg_info=os.path.join(HRU_Folder, "veg_info.csv"),
DEM="#",
)
"""Evaluate total number of HRU, total HRU area and total river length
N_HRU is the total number of HRUs in the HRU polygon
len_Riv is the total river length in the routing network
HRU_Area is the total HRU area in the routing network
"""
### transfer expected product into pandas dataframe
Expect_finalcat_hru_info = Dbf_To_Dataframe(
os.path.join(
Expect_Result_Folder,
"finalcat_hru_info.shp")).sort_values(by=["SubId", "HRU_Area"])
### transfer resulted product into pandas dataframe
Result_finalcat_hru_info = Dbf_To_Dataframe(
os.path.join(
Output_Folder,
"finalcat_hru_info.shp")).sort_values(by=["SubId", "HRU_Area"])
print(
Evaluate_Two_Dataframes(Result_finalcat_hru_info,
Expect_finalcat_hru_info,
Check_Col_NM="HRU_ID"))
assert Evaluate_Two_Dataframes(Result_finalcat_hru_info,
Expect_finalcat_hru_info,
Check_Col_NM="HRU_ID")
shutil.rmtree(Output_Folder)
def test_Select_Routing_product_based_SubId():
"""test function that will:
Function that used to obtain the region of interest from routing
product based on given SubId
"""
###Floder where store the inputs for tests function
Routing_Product_Folder = "./testdata/Routing_product_V2/"
###Folder where store the expected resuts
Expect_Result_Folder = os.path.join("./testdata", "02LE024")
###Folder where the output will be generated
Output_Folder = os.path.join("./testdata", "test_out10")
###Define path of input dataset
###The lake polygons
Path_Con_Lake_ply = os.path.join(
Routing_Product_Folder,
"Con_Lake_Ply.shp") ### Connected lake polygons
Path_NonCon_Lake_ply = os.path.join(
Routing_Product_Folder,
"Non_Con_Lake_Ply.shp") ### None connected lake polygons
###product that need futher process
Path_final_riv_ply = os.path.join(
Routing_Product_Folder, "finalriv_info_ply.shp") ### River polyline
Path_final_riv = os.path.join(Routing_Product_Folder,
"finalriv_info.shp") ### Catchment polygons
###product that do not need need futher process
Path_final_cat_ply = os.path.join(
Routing_Product_Folder, "finalcat_info.shp") ### catchment polygons
Path_final_cat_riv = os.path.join(
Routing_Product_Folder, "finalcat_info_riv.shp") ### CRiver polyline
###Generate test resuts
RTtool = LRRT()
### extract product that do not need further process
RTtool.Select_Routing_product_based_SubId(
OutputFolder=Output_Folder,
Path_Catchment_Polygon=Path_final_cat_ply,
Path_River_Polyline=Path_final_cat_riv,
Path_Con_Lake_ply=Path_Con_Lake_ply,
Path_NonCon_Lake_ply=Path_NonCon_Lake_ply,
mostdownid=1024,
)
### extract product that need further process
subids = RTtool.Select_Routing_product_based_SubId(
OutputFolder=Output_Folder,
Path_Catchment_Polygon=Path_final_riv_ply,
Path_River_Polyline=Path_final_riv,
mostdownid=1024,
)
"""Evaluate extracted shapefiles by comparing attribute table
following shapefiles will be generated by above two commands
'finalcat_info.shp' is the catchment polygons that do not need further process
'finalcat_info_riv.shp' is the river network polyline that do not need further process
'finalriv_info_ply.shp' is the catchment polygons that need further process
'finalriv_info.shp' is the river network polylin that need further process
'Con_Lake_Ply.shp' is the connected lake polygons
'Non_Con_Lake_Ply.shp' is non-connected lake polygons
"""
### transfer resulted catchment polygon that do not need further process into pandas dataframe:Result_Finalcat_info
Result_Finalcat_info = Dbf_To_Dataframe(
os.path.join(Output_Folder, "finalcat_info.shp"))
### transfer resulted river network polyline that do not need further process into pandas dataframe:Result_Finalcat_riv_info
Result_Finalcat_riv_info = Dbf_To_Dataframe(
os.path.join(Output_Folder, "finalcat_info_riv.shp"))
### transfer resulted catchment polygon that need further process into pandas dataframe:Result_Finalriv_info_ply
Result_Finalriv_info_ply = Dbf_To_Dataframe(
os.path.join(Output_Folder, "finalriv_info_ply.shp"))
### transfer resulted river network polyline that need further process into pandas dataframe:Result_Finalcat_riv_info
Result_Finalriv_info = Dbf_To_Dataframe(
os.path.join(Output_Folder, "finalriv_info.shp"))
### transfer resulted connected lake polygon into pandas dataframe Result_Con_Lake_Ply
Result_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Output_Folder, "Con_Lake_Ply.shp"))
### transfer resulted non-connected lake polygon into pandas dataframe Result_Non_Con_Lake_Ply
Result_Non_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Output_Folder, "Non_Con_Lake_Ply.shp"))
### transfer Expect catchment polygon that do not need further process into pandas dataframe:Expect_Finalcat_info
Expect_Finalcat_info = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder, "finalcat_info.shp"))
### transfer Expect river network polyline that do not need further process into pandas dataframe:Expect_Finalcat_riv_info
Expect_Finalcat_riv_info = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder, "finalcat_info_riv.shp"))
### transfer Expect catchment polygon that need further process into pandas dataframe:Expect_Finalriv_info_ply
Expect_Finalriv_info_ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder, "finalriv_info_ply.shp"))
### transfer Expect river network polyline that need further process into pandas dataframe:Expect_Finalriv_info
Expect_Finalriv_info = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder, "finalriv_info.shp"))
### transfer Expect connected lake polygon into pandas dataframe Expect_Con_Lake_Ply
Expect_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder, "Con_Lake_Ply.shp"))
### transfer Expect non-connected lake polygon into pandas dataframe Expect_Non_Con_Lake_Ply
Expect_Non_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder, "Non_Con_Lake_Ply.shp"))
### compare two pd dataframe Result_Finalcat_info and Expect_Finalcat_info
assert Result_Finalcat_info.equals(Expect_Finalcat_info)
### compare two pd dataframe Result_Finalcat_riv_info and Expect_Finalcat_riv_info
assert Result_Finalcat_riv_info.equals(Expect_Finalcat_riv_info)
### compare two pd dataframe Result_Finalriv_info_ply and Expect_Finalriv_info_ply
assert Result_Finalriv_info_ply.equals(Expect_Finalriv_info_ply)
### compare two pd dataframe Result_Finalriv_info and Expect_Finalriv_info
assert Result_Finalriv_info.equals(Expect_Finalriv_info)
### compare two pd dataframe Result_Con_Lake_Ply and Expect_Con_Lake_Ply
assert Result_Con_Lake_Ply.equals(Expect_Con_Lake_Ply)
### compare two pd dataframe Result_Non_Con_Lake_Ply and Expect_Non_Con_Lake_Ply
assert Result_Non_Con_Lake_Ply.equals(Expect_Non_Con_Lake_Ply)
shutil.rmtree(Output_Folder)
def test_Locate_subid_needsbyuser():
"""test function that will:
Function that used to obtain subbasin ID of certain gauge.
or subbasin ID of the polygon that includes the given point
shapefile.
"""
###Floder where store the inputs for tests function
Routing_Product = "./testdata/Routing_product_V2/finalcat_info.shp"
#######
# test for option 1
#######
### Expected generated result
Expect_SubIDs = [1024]
###Define path of input dataset
Gauge_NM = ["02LE024"]
###Generate test resuts
RTtool = LRRT()
subids = RTtool.Locate_subid_needsbyuser(Gauge_NMS=Gauge_NM,
Path_products=Routing_Product)
### compare the length of result suinds and expected result Expect_SubIDs
assert len(subids) == len(Expect_SubIDs)
### compare the content of result suinds and expected result Expect_SubIDs
assert all([a == b for a, b in zip(subids, Expect_SubIDs)])
#######
# test for option 2
#######
### Expected generated result
Expect_SubIDs = [
519,
629,
720,
1024,
999,
884,
883,
1213,
1103,
1200,
1276,
1224,
1276,
1276,
1276,
1277,
1286,
1340,
1347,
1349,
]
###Define path of input dataset
Pointshpfile = "./testdata/Routing_product_V2/obspoint_snap.shp"
###Generate test resuts
subids = RTtool.Locate_subid_needsbyuser(Path_Points=Pointshpfile,
Path_products=Routing_Product)
### compare the length of result suinds and expected result Expect_SubIDs
assert len(subids) == len(Expect_SubIDs)
### compare the content of result suinds and expected result Expect_SubIDs
assert all([a == b for a, b in zip(subids, Expect_SubIDs)])
def test_WatershedDiscretizationToolset():
"""test function that will:
Function that used to Generate a subbasin delineation and river
network using user provied flow accumulation thresthold
without considering lake.
"""
###Floder where store the inputs for tests function
Data_Folder = "./testdata/Required_data_to_start_from_dem/"
###Folder where store the expected resuts
Final_Result_Folder_Expected = os.path.join(
"./testdata", "Final_output_folder", "Expected_InDEM"
)
Temporary_Result_Folder_Expected = os.path.join(
"./testdata", "Temporary_output_folder", "Expected_InDEM"
)
###Folder where the output will be generated
Temporary_Result_Folder_Result = os.path.join(
"./testdata", "Temporary_output_folder", "testout6"
)
Final_Result_Folder_Result = os.path.join(
"./testdata", "Final_output_folder", "testout6"
)
shutil.rmtree(Temporary_Result_Folder_Result, ignore_errors=True)
###The pathes for all inputs
Path_DEM_big = os.path.join(Data_Folder, "DEM_big_merit.tif")
Path_DEM_small = os.path.join(Data_Folder, "DEM_samll_merit.tif")
Path_Lake_ply = os.path.join(Data_Folder, "HyLake.shp")
Path_bkf_wd = os.path.join(Data_Folder, "Bkfullwidth_depth.shp")
Path_Landuse = os.path.join(Data_Folder, "landuse.tif")
Path_Roughness_landuse = os.path.join(Data_Folder, "Landuse_info.csv")
###Generate test resuts
RTtool = LRRT(
dem_in=Path_DEM_small,
WidDep=Path_bkf_wd,
Lakefile=Path_Lake_ply,
Landuse=Path_Landuse,
Landuseinfo=Path_Roughness_landuse,
OutputFolder=Final_Result_Folder_Result,
TempOutFolder=Temporary_Result_Folder_Result,
)
### test using extent of input dem as processing extent
RTtool.Generatmaskregion()
RTtool.Generateinputdata()
RTtool.WatershedDiscretizationToolset(accthresold=500)
"""Evaluate raster cat1
cat1 is a raster represent the delineated subbasins without
considering lakes
"""
### transfer expected raster cat1 into np array Expected_cat1_Array
Expected_cat1_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="cat1",
)
### transfer resulted raster cat1 into np array Result_cat1_Array
Result_cat1_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="cat1",
)
### compare two Expected_Mask_Array and Result_Mask_Array
assert (Expected_cat1_Array == Result_cat1_Array).all()
"""Evaluate raster str_grass_r
str_grass_r is a river network in raster format
"""
### transfer expected raster str_grass_r into np array Expected_str_Array
Expected_str_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="str_grass_r",
)
### transfer resulted raster str_grass_r into np array Result_str_Array
Result_str_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="str_grass_r",
)
### compare two Expected_str_Array and Result_str_Array
assert (Expected_str_Array == Result_str_Array).all()
"""Evaluate raster Connect_Lake
Connect_Lake is the lake raster only contain lakes connected by
str_grass_r
"""
### transfer expected raster Connect_Lake into np array Expected_CLake_Array
Expected_CLake_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="Connect_Lake",
)
### transfer resulted raster Connect_Lake into np array Result_CLake_Array
Result_CLake_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="Connect_Lake",
)
### compare two Expected_CLake_Array and Result_CLake_Array
# assert (Expected_CLake_Array == Result_CLake_Array).all()
"""Evaluate raster Nonconnect_Lake
Nonconnect_Lake is the lake raster only contain lakes not connected by
str_grass_r
"""
### transfer expected raster Nonconnect_Lake into np array Expected_NCLake_Array
Expected_NCLake_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="Nonconnect_Lake",
)
### transfer resulted raster Nonconnect_Lake into np array Result_NCLake_Array
Result_NCLake_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="Nonconnect_Lake",
)
### compare two Expected_NCLake_Array and Result_NCLake_Array
# assert (Expected_NCLake_Array == Result_NCLake_Array).all()
RTtool.Output_Clean()
def test_Customize_Routing_Topology():
"""test function that will:
Function that used to simplify the routing product by
using user provided minimum subbasin drainage area.
The input catchment polygons are routing product before
merging for lakes. It is provided with routing product.
The result is the simplified catchment polygons. But
result from this fuction still not merging catchment
covering by the same lake. Thus, The result generated
from this tools need further processed by
Define_Final_Catchment, or can be further processed by
SelectLakes
"""
###Floder where store the inputs for tests function
Routing_Product_Folder = "./testdata/02LE024/"
###Folder where store the expected resuts
Expect_Result_Folder = os.path.join("./testdata", "Simplified_By_DA")
###Folder where the output will be generated
Output_Folder = os.path.join("./testdata", "testout10")
###The pathes for all inputs
Path_Con_Lake_ply = os.path.join(
Routing_Product_Folder,
"Con_Lake_Ply.shp") ### Connected lake polygons
Path_NonCon_Lake_ply = os.path.join(
Routing_Product_Folder,
"Non_Con_Lake_Ply.shp") ### None connected lake polygons
Path_final_riv_ply = os.path.join(
Routing_Product_Folder, "finalriv_info_ply.shp") ### River polyline
Path_final_riv = os.path.join(Routing_Product_Folder,
"finalriv_info.shp") ### Catchment polygons
###Generate test resuts
Area_Min = 60 ## minimum catchment drainage area.
RTtool = LRRT()
RTtool.Customize_Routing_Topology(
Path_final_riv_ply=Path_final_riv_ply,
Path_final_riv=Path_final_riv,
Path_Con_Lake_ply=Path_Con_Lake_ply,
Path_NonCon_Lake_ply=Path_NonCon_Lake_ply,
Area_Min=Area_Min,
OutputFolder=Output_Folder,
)
"""Evaluate total number of subbasin, total subbasin area and total river length
N_Cat is the total number of subbasins in the simplified routing network
len_Riv is the total river length in the simplified routing network
Bas_Area is the total subbasin area in the simplified routing network
"""
### transfer expected siplified product into pandas dataframe
Expect_Finalriv_info_ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder,
"finalriv_info_ply.shp")).sort_values(by=["SubId"])
### calcuate expected total number of catchment:Expect_N_Cat
Expect_N_Cat = len(Expect_Finalriv_info_ply)
### calcuate expected total river length :Expect_len_Riv
Expect_len_Riv = sum(Expect_Finalriv_info_ply["RivLength"])
### calcuate expected total basin area :Expect_Bas_Area
Expect_Bas_Area = sum(Expect_Finalriv_info_ply["BasArea"])
### transfer resulted siplified product into pandas dataframe
Result_Finalriv_info_ply = Dbf_To_Dataframe(
os.path.join(Output_Folder,
"finalriv_info_ply.shp")).sort_values(by=["SubId"])
### calcuate resulted total number of catchment:Result_N_Cat
Result_N_Cat = len(Result_Finalriv_info_ply)
### calcuate resulted total river length :Result_len_Riv
Result_len_Riv = sum(Result_Finalriv_info_ply["RivLength"])
### calcuate resulted total basin area :Result_Bas_Area
Result_Bas_Area = sum(Result_Finalriv_info_ply["BasArea"])
### compare Expect_N_Cat and Result_N_Cat
assert Expect_N_Cat == Result_N_Cat
### compare Expect_len_Riv and Result_len_Riv
assert Expect_len_Riv == pytest.approx(Result_len_Riv, 0.1)
### compare Expect_Bas_Area and Result_Bas_Area
assert Expect_Bas_Area == pytest.approx(Result_Bas_Area, 0.1)
"""Evaluate lake polygon files
Con_Lake_Ply is the lake polygon that connected by river network
Non_Con_Lake_Ply is the lake polygon that did not connected by
river network
"""
### transfer expected siplified connected lake polygon into pandas dataframe Expect_Con_Lake_Ply
Expect_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder, "Con_Lake_Ply.shp"))
### transfer expected siplified non connected lake polygon into pandas dataframe Expect_Non_Con_Lake_Ply
Expect_Non_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder, "Non_Con_Lake_Ply.shp"))
### transfer resulted siplified connected lake polygon into pandas dataframe Result_Con_Lake_Ply
Result_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Output_Folder, "Con_Lake_Ply.shp"))
### transfer resulted siplified non connected lake polygon into pandas dataframe Result_Non_Con_Lake_Ply
Result_Non_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Output_Folder, "Non_Con_Lake_Ply.shp"))
### compare two pandas dataframe Expect_Con_Lake_Ply and Result_Con_Lake_Ply
assert Result_Con_Lake_Ply.equals(Expect_Con_Lake_Ply)
### compare two pandas dataframe Expect_Non_Con_Lake_Ply and Result_Non_Con_Lake_Ply
assert Result_Non_Con_Lake_Ply.equals(Expect_Non_Con_Lake_Ply)
shutil.rmtree(Output_Folder)
def test_SelectLakes():
"""test function that will:
Function that used to simplify the routing product by user
provided lake area thresthold.
The input catchment polygons is the routing product before
merging for lakes. It is provided with the routing product.
The result is the simplified catchment polygons. But
result from this fuction still not merging catchment
covering by the same lake. Thus, The result generated
from this tools need further processed by
Define_Final_Catchment, or can be further processed by
Customize_Routing_Topology
"""
###Floder where store the inputs for tests function
Routing_Product_Folder = "./testdata/Simplified_By_DA/"
###Folder where store the expected resuts
Expect_Result_Folder = os.path.join("./testdata", "Simplify_By_Lake_Area")
###Folder where the output will be generated
Output_Folder = os.path.join("./testdata", "testout10")
###The pathes for all inputs
Path_Con_Lake_ply = os.path.join(
Routing_Product_Folder,
"Con_Lake_Ply.shp") ### Connected lake polygons
Path_NonCon_Lake_ply = os.path.join(
Routing_Product_Folder,
"Non_Con_Lake_Ply.shp") ### None connected lake polygons
Path_final_riv_ply = os.path.join(
Routing_Product_Folder, "finalriv_info_ply.shp") ### River polyline
Path_final_riv = os.path.join(Routing_Product_Folder,
"finalriv_info.shp") ### Catchment polygons
###Generate test resuts
Lake_Area_thresthold_Connected_Lake = 10
Lake_Area_thresthold_NonConnected_Lake = 3
RTtool = LRRT()
RTtool.SelectLakes(
Path_final_riv_ply=Path_final_riv_ply,
Path_final_riv=Path_final_riv,
Path_Con_Lake_ply=Path_Con_Lake_ply,
Path_NonCon_Lake_ply=Path_NonCon_Lake_ply,
Thres_Area_Conn_Lakes=Lake_Area_thresthold_Connected_Lake,
Thres_Area_Non_Conn_Lakes=Lake_Area_thresthold_NonConnected_Lake,
Selection_Method="ByArea",
OutputFolder=Output_Folder,
)
"""Evaluate result attribute table of result polygon,polylie, and two
lake polygons
"""
### transfer expected siplified product into pandas dataframe
Expect_Finalriv_info_ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder,
"finalriv_info_ply.shp")).sort_values(by=["SubId"])
### transfer resulted siplified product into pandas dataframe
Result_Finalriv_info_ply = Dbf_To_Dataframe(
os.path.join(Output_Folder,
"finalriv_info_ply.shp")).sort_values(by=["SubId"])
assert Evaluate_Two_Dataframes(Result_Finalriv_info_ply,
Expect_Finalriv_info_ply,
Check_Col_NM="SubId")
"""Evaluate lake polygon files
Con_Lake_Ply is the lake polygon that connected by river network
Non_Con_Lake_Ply is the lake polygon that did not connected by
river network
"""
### transfer expected siplified connected lake polygon into pandas dataframe Expect_Con_Lake_Ply
Expect_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder,
"Con_Lake_Ply.shp")).sort_values(by=["Hylak_id"])
### transfer expected siplified non connected lake polygon into pandas dataframe Expect_Non_Con_Lake_Ply
Expect_Non_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Expect_Result_Folder,
"Non_Con_Lake_Ply.shp")).sort_values(by=["Hylak_id"])
### transfer resulted siplified connected lake polygon into pandas dataframe Result_Con_Lake_Ply
Result_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Output_Folder,
"Con_Lake_Ply.shp")).sort_values(by=["Hylak_id"])
### transfer resulted siplified non connected lake polygon into pandas dataframe Result_Non_Con_Lake_Ply
Result_Non_Con_Lake_Ply = Dbf_To_Dataframe(
os.path.join(Output_Folder,
"Non_Con_Lake_Ply.shp")).sort_values(by=["Hylak_id"])
### compare two pandas dataframe Expect_Con_Lake_Ply and Result_Con_Lake_Ply
assert Evaluate_Two_Dataframes(Expect_Con_Lake_Ply,
Result_Con_Lake_Ply,
Check_Col_NM="Hylak_id")
### compare two pandas dataframe Expect_Non_Con_Lake_Ply and Result_Non_Con_Lake_Ply
assert Evaluate_Two_Dataframes(Expect_Non_Con_Lake_Ply,
Result_Non_Con_Lake_Ply,
Check_Col_NM="Hylak_id")
shutil.rmtree(Output_Folder)
def test_GenerateRavenInput(HYDAT_Path):
"""test function that will:
Function that used to generate Raven input files. All output
will be stored in folder "<OutputFolder>/RavenInput".
Parameters
----------
CA_HYDAT (string): Path and filename of previously downloaded
external database containing streamflow observations,
e.g. HYDAT for Canada ("Hydat.sqlite3").
Read from command line
"""
###Floder where store the inputs for tests function
Routing_Product_Folder = "./testdata/Simplified_By_DA/"
###Folder where store the expected resuts
Expect_Result_Folder = os.path.join("./testdata", "Simplified_By_DA")
###Folder where the output will be generated
Output_Folder = os.path.join("./testdata", "testout10")
###Define path of input dataset
CA_HYDAT = HYDAT_Path
Path_final_hru_info = os.path.join(
Routing_Product_Folder, "finalcat_hru_info.shp") ### River polyline
###Generate test resuts
RTtool = LRRT()
RTtool.GenerateRavenInput(
Path_final_hru_info=Path_final_hru_info,
OutputFolder=Output_Folder,
Model_Name="Mytest_model",
WriteObsrvt=True,
Startyear=2010,
EndYear=2014,
CA_HYDAT=CA_HYDAT,
WarmUp=1,
)
"""Evaluate generated raven input files in result and expected folder
Mytest_model.rvh contains subbasins and HRUs
channel_properties.rvp contains definition and parameters for channels
Lakes.rvh contains definition and parameters of lakes
02LE013_720.rvt contains downloaded streamflow observation data
02LE015_519.rvt contains downloaded streamflow observation data
02LE024_1024.rvt contains downloaded streamflow observation data
obsinfo.csv contains information of each gauge such as number of missing
value and drainage area.
"""
assert filecmp.cmp(
os.path.join(Output_Folder, "RavenInput", "Mytest_model.rvh"),
os.path.join(Expect_Result_Folder, "RavenInput", "Mytest_model.rvh"),
)
assert filecmp.cmp(
os.path.join(Output_Folder, "RavenInput", "channel_properties.rvp"),
os.path.join(Expect_Result_Folder, "RavenInput",
"channel_properties.rvp"),
)
assert filecmp.cmp(
os.path.join(Output_Folder, "RavenInput", "Lakes.rvh"),
os.path.join(Expect_Result_Folder, "RavenInput", "Lakes.rvh"),
)
assert filecmp.cmp(
os.path.join(Output_Folder, "RavenInput", "obs", "02LE013_720.rvt"),
os.path.join(Expect_Result_Folder, "RavenInput", "obs",
"02LE013_720.rvt"),
)
assert filecmp.cmp(
os.path.join(Output_Folder, "RavenInput", "obs", "02LE015_519.rvt"),
os.path.join(Expect_Result_Folder, "RavenInput", "obs",
"02LE015_519.rvt"),
)
assert filecmp.cmp(
os.path.join(Output_Folder, "RavenInput", "obs", "02LE024_1024.rvt"),
os.path.join(Expect_Result_Folder, "RavenInput", "obs",
"02LE024_1024.rvt"),
)
assert filecmp.cmp(
os.path.join(Output_Folder, "RavenInput", "obs", "obsinfo.csv"),
os.path.join(Expect_Result_Folder, "RavenInput", "obs", "obsinfo.csv"),
)
RTtool.Output_Clean()
def test_Generatmaskregion():
"""test function that will:
Define the processing extent. Function that used to define
processing spatial extent (PSE). The processing spatial extent
is a region where Toolbox will work in. Toolbox will not
process grids or features outside the processing spatial extent.
"""
###Floder where store the inputs for tests function
Data_Folder = "./testdata/Required_data_to_start_from_dem/"
###Folder where store the expected resuts
Final_Result_Folder_Expected = os.path.join(
"./testdata", "Final_output_folder", "Expected_InDEM"
)
Temporary_Result_Folder_Expected = os.path.join(
"./testdata", "Temporary_output_folder", "Expected_InDEM"
)
###Folder where the output will be generated
Temporary_Result_Folder_Result = os.path.join(
"./testdata", "Temporary_output_folder", "testout3"
)
Final_Result_Folder_Result = os.path.join(
"./testdata", "Final_output_folder", "testout3"
)
###The pathes for all inputs
Path_DEM_big = os.path.join(Data_Folder, "DEM_big_merit.tif")
Path_DEM_small = os.path.join(Data_Folder, "DEM_samll_merit.tif")
Path_Lake_ply = os.path.join(Data_Folder, "HyLake.shp")
Path_bkf_wd = os.path.join(Data_Folder, "Bkfullwidth_depth.shp")
Path_Landuse = os.path.join(Data_Folder, "landuse.tif")
Path_Roughness_landuse = os.path.join(Data_Folder, "Landuse.csv")
###Generate test resuts, for option 1
RTtool = LRRT(
dem_in=Path_DEM_small,
WidDep=Path_bkf_wd,
Lakefile=Path_Lake_ply,
Landuse=Path_Landuse,
Landuseinfo=Path_Roughness_landuse,
OutputFolder=Final_Result_Folder_Result,
TempOutFolder=Temporary_Result_Folder_Result,
)
RTtool.Generatmaskregion()
"""Evaluate raster MASK
MASK is a mask raster stored in grass database, which indicate
the spatial processing extent.
"""
### transfer expected raster alllake into np array Expected_Mask_Array
Expected_Mask_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="MASK",
)
### transfer resulted raster alllake into np array Result_Mask_Array
Result_Mask_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="MASK",
)
### compare two Expected_Mask_Array and Result_Mask_Array
assert (Expected_Mask_Array == Result_Mask_Array).all()
### test another option to define processing extent
### Folder where the output will be generated
Final_Result_Folder_Expected = os.path.join(
"./testdata", "Final_output_folder", "Expected_Out_Cord"
)
Temporary_Result_Folder_Expected = os.path.join(
"./testdata", "Temporary_output_folder", "Expected_Out_Cord"
)
###Folder where the output will be generated
Temporary_Result_Folder_Result = os.path.join(
"./testdata", "Temporary_output_folder", "testout4"
)
Final_Result_Folder_Result = os.path.join(
"./testdata", "Final_output_folder", "testout4"
)
RTtool = LRRT(
dem_in=Path_DEM_small,
WidDep=Path_bkf_wd,
Lakefile=Path_Lake_ply,
Landuse=Path_Landuse,
Landuseinfo=Path_Roughness_landuse,
OutputFolder=Final_Result_Folder_Result,
TempOutFolder=Temporary_Result_Folder_Result,
)
OutletPointxy = [-92.387, 49.09]
RTtool.Generatmaskregion(OutletPoint=OutletPointxy)
"""Evaluate raster MASK
MASK is a mask raster stored in grass database, which indicate
the spatial processing extent.
"""
### transfer expected raster alllake into np array Expected_Mask_Array
Expected_Mask_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="MASK",
)
### transfer resulted raster alllake into np array Result_Mask_Array
Result_Mask_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="MASK",
)
### compare two Expected_Mask_Array and Result_Mask_Array
assert (Expected_Mask_Array == Result_Mask_Array).all()
def test_AutomatedWatershedsandLakesFilterToolset():
"""test function that will:
Add lake inflow and outflow points as new subabsin outlet
"""
###Floder where store the inputs for tests functions
Data_Folder = "./testdata/Required_data_to_start_from_dem/"
###Folder where store the expected resuts
Final_Result_Folder_Expected = os.path.join(
"./testdata", "Final_output_folder", "Expected_InDEM"
)
Temporary_Result_Folder_Expected = os.path.join(
"./testdata", "Temporary_output_folder", "Expected_InDEM"
)
###Folder where the output will be generated
Temporary_Result_Folder_Result = os.path.join(
"./testdata", "Temporary_output_folder", "testout1"
)
Final_Result_Folder_Result = os.path.join(
"./testdata", "Final_output_folder", "testout1"
)
shutil.rmtree(Temporary_Result_Folder_Result, ignore_errors=True)
###Define path of input dataset
Path_DEM_small = os.path.join(Data_Folder, "DEM_samll_merit.tif")
Path_Lake_ply = os.path.join(Data_Folder, "HyLake.shp")
Path_bkf_wd = os.path.join(Data_Folder, "Bkfullwidth_depth.shp")
Path_Landuse = os.path.join(Data_Folder, "landuse.tif")
Path_Roughness_landuse = os.path.join(Data_Folder, "Landuse.csv")
###Generate test resuts
RTtool = LRRT(
dem_in=Path_DEM_small,
WidDep=Path_bkf_wd,
Lakefile=Path_Lake_ply,
Landuse=Path_Landuse,
Landuseinfo=Path_Roughness_landuse,
OutputFolder=Final_Result_Folder_Result,
TempOutFolder=Temporary_Result_Folder_Result,
)
RTtool.Generatmaskregion()
RTtool.Generateinputdata()
RTtool.WatershedDiscretizationToolset(accthresold=500)
RTtool.AutomatedWatershedsandLakesFilterToolset(
Thre_Lake_Area_Connect=0, Thre_Lake_Area_nonConnect=0
)
"""Evaluate raster Net_cat
Net_cat is a subbsin raster after adding lake inlet and outlet as
additional subbasin outlet.
"""
### transfer expected raster Net_cat into np array Expected_Net_cat_Array
Expected_Net_cat_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="Net_cat",
)
### transfer test raster Net_cat into np array Result_Net_cat_Array
Result_Net_cat_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="Net_cat",
)
### compare two Expected_Net_cat_Array and Result_Net_cat_Array
assert (Expected_Net_cat_Array == Result_Net_cat_Array).all()
"""Evaluate raster ndir_grass
ndir_grass is a raster represent modified flow direction
"""
### transfer expected raster ndir_grass into np array Expected_ndir_Array
Expected_ndir_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="ndir_grass",
)
### transfer test raster ndir_grass into np array Result_ndir_Array
Result_ndir_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="ndir_grass",
)
### compare two Expected_ndir_Array and Result_ndir_Array
assert (Expected_ndir_Array == Result_ndir_Array).all()
### clean test output folder
RTtool.Output_Clean()
def test_Generateinputdata():
"""test function that will:
Preprocessing input dataset, Function that used to project and clip
input dataset such as DEM, Land use, Lake polygon etc with defined
processing extent by function Generatmaskregion. And then it will
rasterize these vector files.
"""
###Floder where store the inputs for tests function
Data_Folder = "./testdata/Required_data_to_start_from_dem/"
###Folder where store the expected resuts
Final_Result_Folder_Expected = os.path.join(
"./testdata", "Final_output_folder", "Expected_InDEM"
)
Temporary_Result_Folder_Expected = os.path.join(
"./testdata", "Temporary_output_folder", "Expected_InDEM"
)
###Folder where the output will be generated
Temporary_Result_Folder_Result = os.path.join(
"./testdata", "Temporary_output_folder", "testout2"
)
Final_Result_Folder_Result = os.path.join(
"./testdata", "Final_output_folder", "testout2"
)
shutil.rmtree(Temporary_Result_Folder_Result, ignore_errors=True)
###Define path of input dataset
Path_DEM_big = os.path.join(Data_Folder, "DEM_big_merit.tif")
Path_DEM_small = os.path.join(Data_Folder, "DEM_samll_merit.tif")
Path_Lake_ply = os.path.join(Data_Folder, "HyLake.shp")
Path_bkf_wd = os.path.join(Data_Folder, "Bkfullwidth_depth.shp")
Path_Landuse = os.path.join(Data_Folder, "landuse.tif")
Path_Roughness_landuse = os.path.join(Data_Folder, "Landuse_info.csv")
###Generate test resuts
RTtool = LRRT(
dem_in=Path_DEM_small,
WidDep=Path_bkf_wd,
Lakefile=Path_Lake_ply,
Landuse=Path_Landuse,
Landuseinfo=Path_Roughness_landuse,
OutputFolder=Final_Result_Folder_Result,
TempOutFolder=Temporary_Result_Folder_Result,
)
RTtool.Generatmaskregion()
RTtool.Generateinputdata()
"""Evaluate raster alllake
alllake is a lake raster which include all lakes
"""
### transfer expected raster alllake into np array Expected_alllake_Array
Expected_alllake_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="alllake",
)
### transfer resulted raster alllake into np array Result_alllake_Array
Result_alllake_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="alllake",
)
### compare two Expected_alllake_Array and Result_alllake_Array
assert (Expected_alllake_Array == Result_alllake_Array).all()
"""Evaluate raster Lake_Bound
Lake_Bound is a raster represent the lake boundary grids
"""
### transfer expected raster Lake_Bound into np array Expected_LBound_Array
Expected_LBound_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="Lake_Bound",
)
### transfer resulted raster Lake_Bound into np array Result_LBound_Array
Result_LBound_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="Lake_Bound",
)
### compare two Expected_LBound_Array and Result_LBound_Array
assert (Expected_LBound_Array == Result_LBound_Array).all()
"""Evaluate raster acc_grass
acc_grass is the flow accumulation raster generated by 'r.watershed'
"""
### transfer expected raster acc_grass into np array Expected_acc_Array
Expected_acc_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Expected, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="acc_grass",
)
### transfer resulted raster acc_grass into np array Result_acc_Array
Result_acc_Array = Return_Raster_As_Array(
grassdb=os.path.join(Temporary_Result_Folder_Result, "grassdata_toolbox"),
grass_location="Geographic",
raster_mn="acc_grass",
)
### compare two Expected_acc_Array and Result_acc_Array
assert (Expected_acc_Array == Result_acc_Array).all()
### clean test output folder
RTtool.Output_Clean()