def test_no_distances(self):
"""WindEnergy: testing case w/ AOI and land poly, but w/o distances."""
from natcap.invest import wind_energy
from natcap.invest.utils import _assert_vectors_equal
args = WindEnergyRegressionTests.generate_base_args(self.workspace_dir)
args['aoi_vector_path'] = os.path.join(SAMPLE_DATA,
'New_England_US_Aoi.shp')
args['land_polygon_vector_path'] = os.path.join(
SAMPLE_DATA, 'simple_north_america_polygon.shp')
wind_energy.execute(args)
raster_results = [
'density_W_per_m2.tif', 'harvested_energy_MWhr_per_yr.tif'
]
for raster_path in raster_results:
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output', raster_path))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, 'nodistances', raster_path))
numpy.testing.assert_allclose(model_array, reg_array)
vector_path = 'wind_energy_points.shp'
_assert_vectors_equal(
os.path.join(args['workspace_dir'], 'output', vector_path),
os.path.join(REGRESSION_DATA, 'nodistances', vector_path))
def test_collect_multipart_gdal_raster(self):
"""Datastack: test collect multipart gdal raster."""
from natcap.invest import datastack
params = {
'raster': os.path.join(DATA_DIR, 'dem'),
}
# Collect the raster's files into a single archive
archive_path = os.path.join(self.workspace, 'archive.invs.tar.gz')
datastack.build_datastack_archive(params, 'sample_model', archive_path)
# extract the archive
out_directory = os.path.join(self.workspace, 'extracted_archive')
with tarfile.open(archive_path) as tar:
tar.extractall(out_directory)
archived_params = json.load(
open(
os.path.join(out_directory,
datastack.DATASTACK_PARAMETER_FILENAME)))['args']
self.assertEqual(len(archived_params), 1)
model_array = pygeoprocessing.raster_to_numpy_array(params['raster'])
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(out_directory, archived_params['raster']))
numpy.testing.assert_allclose(model_array, reg_array)
def test_no_aoi(self):
"""WindEnergy: testing base case w/o AOI, distances, or valuation."""
from natcap.invest import wind_energy
from natcap.invest.utils import _assert_vectors_equal
args = WindEnergyRegressionTests.generate_base_args(self.workspace_dir)
# Also test on input bathymetry that has equal x, y pixel sizes
args['bathymetry_path'] = os.path.join(
SAMPLE_DATA, 'resampled_global_dem_equal_pixel.tif')
wind_energy.execute(args)
raster_results = [
'density_W_per_m2.tif', 'harvested_energy_MWhr_per_yr.tif'
]
for raster_path in raster_results:
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output', raster_path))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, 'noaoi', raster_path))
numpy.testing.assert_allclose(model_array, reg_array)
vector_path = 'wind_energy_points.shp'
_assert_vectors_equal(
os.path.join(args['workspace_dir'], 'output', vector_path),
os.path.join(REGRESSION_DATA, 'noaoi', vector_path))
def test_globio_shape_infra(self):
"""GLOBIO: regression testing with shapefile infrastructure."""
from natcap.invest import globio
args = {
'aoi_path': '',
'globio_lulc_path': os.path.join(
SAMPLE_DATA, 'globio_lulc_small.tif'),
'infrastructure_dir': os.path.join(
SAMPLE_DATA, 'shape_infrastructure'),
'intensification_fraction': '0.46',
'msa_parameters_path': os.path.join(
SAMPLE_DATA, 'msa_parameters.csv'),
'predefined_globio': True,
'workspace_dir': self.workspace_dir,
'n_workers': '-1',
}
globio.execute(args)
GLOBIOTests._test_same_files(
os.path.join(REGRESSION_DATA, 'expected_file_list_lulc.txt'),
args['workspace_dir'])
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'msa.tif'))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, 'msa_shape_infra_regression.tif'))
numpy.testing.assert_allclose(model_array, reg_array)
def test_raster_average(self):
"""Stormwater: test raster_average function."""
from natcap.invest import stormwater
array = numpy.empty((150, 150))
nodata = -1
array[:, 0:128] = 10
array[:, 128:149] = 20
array[:, 149] = nodata
data_path = os.path.join(self.workspace_dir, 'data.tif')
kernel_path = os.path.join(self.workspace_dir, 'kernel.tif')
average_path = os.path.join(self.workspace_dir, 'average.tif')
to_raster(array, data_path, pixel_size=(10, -10))
stormwater.raster_average(data_path, 11, kernel_path, average_path)
expected_kernel = numpy.array([[0, 1, 0], [1, 1, 1], [0, 1, 0]],
dtype=numpy.uint8)
actual_kernel = pygeoprocessing.raster_to_numpy_array(kernel_path)
numpy.testing.assert_equal(actual_kernel, expected_kernel)
actual_average = pygeoprocessing.raster_to_numpy_array(average_path)
expected_average = numpy.empty((150, 150))
expected_average[:, 0:127] = 10
expected_average[:, 127] = 12
expected_average[0, 127] = 12.5
expected_average[-1, 127] = 12.5
expected_average[:, 128] = 18
expected_average[0, 128] = 17.5
expected_average[-1, 128] = 17.5
expected_average[:, 129:149] = 20
expected_average[:, 149] = -1
numpy.testing.assert_allclose(actual_average, expected_average)
def test_no_aoi_or_val(self):
"""WaveEnergy: testing Biophysical component w/o AOI or valuation."""
from natcap.invest import wave_energy
args = WaveEnergyRegressionTests.generate_base_args(self.workspace_dir)
wave_energy.execute(args)
raster_results = [
'wp_rc.tif', 'wp_kw.tif', 'capwe_rc.tif', 'capwe_mwh.tif'
]
for raster_path in raster_results:
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output', raster_path))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, 'noaoi', raster_path))
numpy.testing.assert_allclose(model_array, reg_array)
table_results = ['capwe_rc.csv', 'wp_rc.csv']
for table_path in table_results:
model_df = pandas.read_csv(
os.path.join(args['workspace_dir'], 'output', table_path))
reg_df = pandas.read_csv(
os.path.join(REGRESSION_DATA, 'noaoi', table_path))
pandas.testing.assert_frame_equal(model_df, reg_df)
def test_collect_rasters(self):
"""Datastack: test collect GDAL rasters."""
from natcap.invest import datastack
for raster_filename in (
'dem', # This is a multipart raster
'landcover.tif'): # This is a single-file raster
params = {
'raster': os.path.join(DATA_DIR, raster_filename),
}
# Collect the raster's files into a single archive
archive_path = os.path.join(self.workspace, 'archive.invs.tar.gz')
datastack.build_datastack_archive(params,
'test_datastack_modules.raster',
archive_path)
# extract the archive
out_directory = os.path.join(self.workspace, 'extracted_archive')
with tarfile.open(archive_path) as tar:
tar.extractall(out_directory)
archived_params = json.load(
open(
os.path.join(
out_directory,
datastack.DATASTACK_PARAMETER_FILENAME)))['args']
self.assertEqual(len(archived_params), 1)
model_array = pygeoprocessing.raster_to_numpy_array(
params['raster'])
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(out_directory, archived_params['raster']))
numpy.testing.assert_allclose(model_array, reg_array)
def test_hra_no_subregion_multiplicative_exponential(self):
"""HRA: regression testing with exponential, multiplicative eqn."""
import natcap.invest.hra
args = HraRegressionTests.generate_base_args(self.workspace_dir)
_make_info_csv(args['info_table_path'], self.workspace_dir)
_make_criteria_csv(args['criteria_table_path'], self.workspace_dir)
_make_aoi_vector(args['aoi_vector_path'])
args['risk_eq'] = 'Multiplicative'
args['decay_eq'] = 'Exponential'
args['resolution'] = 1
aoi_vector_path = os.path.join(
self.workspace_dir, 'no_subregion_aoi.shp')
# Test if ``Total Region`` gets written in output stats CSV
_make_aoi_vector(aoi_vector_path, subregion_field=False)
args['aoi_vector_path'] = aoi_vector_path
natcap.invest.hra.execute(args)
output_rasters = [
'TOTAL_RISK_habitat_0', 'TOTAL_RISK_habitat_1',
'TOTAL_RISK_Ecosystem', 'RECLASS_RISK_habitat_0',
'RECLASS_RISK_habitat_1', 'RECLASS_RISK_Ecosystem']
# Assert rasters are equal
output_raster_paths = [
os.path.join(self.workspace_dir, 'outputs', raster_name + '.tif')
for raster_name in output_rasters]
expected_raster_paths = [os.path.join(
TEST_DATA, raster_name + '_mul_exp.tif') for raster_name in
output_rasters]
# Append a intermediate raster to test the linear decay equation
output_raster_paths.append(
os.path.join(self.workspace_dir, 'intermediate_outputs',
'C_habitat_0_stressor_1.tif'))
expected_raster_paths.append(
os.path.join(TEST_DATA, 'C_habitat_0_stressor_1_mul_exp.tif'))
for output_raster, expected_raster in zip(
output_raster_paths, expected_raster_paths):
model_array = pygeoprocessing.raster_to_numpy_array(output_raster)
reg_array = pygeoprocessing.raster_to_numpy_array(expected_raster)
numpy.testing.assert_allclose(model_array, reg_array)
# Assert summary statistics CSV equal
output_csv_path = os.path.join(
self.workspace_dir, 'outputs', 'SUMMARY_STATISTICS.csv')
expected_csv_path = os.path.join(
TEST_DATA, 'SUMMARY_STATISTICS_mul_exp.csv')
model_df = pandas.read_csv(output_csv_path)
reg_df = pandas.read_csv(expected_csv_path)
pandas.testing.assert_frame_equal(model_df, reg_df)
def test_val_gridpts_windprice(self):
"""WindEnergy: testing Valuation w/ grid pts and wind price."""
from natcap.invest import wind_energy
from natcap.invest.utils import _assert_vectors_equal
args = WindEnergyRegressionTests.generate_base_args(self.workspace_dir)
args['aoi_vector_path'] = os.path.join(SAMPLE_DATA,
'New_England_US_Aoi.shp')
args['land_polygon_vector_path'] = os.path.join(
SAMPLE_DATA, 'simple_north_america_polygon.shp')
args['min_distance'] = 0
args['max_distance'] = 200000
args['valuation_container'] = True
args['foundation_cost'] = 2
args['discount_rate'] = 0.07
# Test that only grid points are provided in grid_points_path
args['grid_points_path'] = os.path.join(SAMPLE_DATA,
'resampled_grid_pts.csv')
args['price_table'] = False
args['wind_price'] = 0.187
args['rate_change'] = 0.2
wind_energy.execute(args)
# Make sure the output files were created.
vector_path = 'wind_energy_points.shp'
self.assertTrue(
os.path.exists(
os.path.join(args['workspace_dir'], 'output', vector_path)))
# Run through the model again, which should mean deleting shapefiles
# that have already been made, but which need to be created again.
wind_energy.execute(args)
raster_results = [
'carbon_emissions_tons.tif', 'levelized_cost_price_per_kWh.tif',
'npv_US_millions.tif'
]
for raster_path in raster_results:
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output', raster_path))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, 'pricevalgrid', raster_path))
numpy.testing.assert_allclose(model_array, reg_array)
vector_path = 'wind_energy_points.shp'
_assert_vectors_equal(
os.path.join(args['workspace_dir'], 'output', vector_path),
os.path.join(REGRESSION_DATA, 'pricevalgrid', vector_path))
def test_is_near(self):
"""Stormwater: test is_near function."""
from natcap.invest import stormwater
is_connected_array = numpy.array(
[[0, 0, 1, 0, 0, 0], [1, 0, 1, 0, 0, 0], [0, 0, 0, 0, 0, 1]],
dtype=numpy.uint8)
radius = 1 # 1 pixel
# search kernel:
# [0, 1, 0],
# [1, 1, 1],
# [0, 1, 0]
# convolution sum array:
# [1, 1, 2, 1, 0, 0],
# [1, 1, 2, 1, 0, 1],
# [1, 0, 1, 0, 1, 1]
# expected is_near array: sum > 0
expected = numpy.array(
[[1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 0, 1], [1, 0, 1, 0, 1, 1]],
dtype=numpy.uint8)
connected_path = os.path.join(self.workspace_dir, 'connected.tif')
distance_path = os.path.join(self.workspace_dir, 'distance.tif')
out_path = os.path.join(self.workspace_dir, 'near_connected.tif')
to_raster(is_connected_array, connected_path, pixel_size=(10, -10))
mocked = functools.partial(mock_iterblocks,
yoffs=[0],
ysizes=[3],
xoffs=[0, 3],
xsizes=[3, 3])
with mock.patch('natcap.invest.stormwater.pygeoprocessing.iterblocks',
mocked):
stormwater.is_near(connected_path, radius, distance_path, out_path)
actual = pygeoprocessing.raster_to_numpy_array(out_path)
numpy.testing.assert_equal(expected, actual)
def test_flow_dir_d8(self):
"""PGP.routing: test D8 flow."""
dem_path = os.path.join(self.workspace_dir, 'dem.tif')
dem_array = numpy.zeros((11, 11), dtype=numpy.float32)
_array_to_raster(dem_array, None, dem_path)
target_flow_dir_path = os.path.join(self.workspace_dir, 'flow_dir.tif')
pygeoprocessing.routing.flow_dir_d8((dem_path, 1),
target_flow_dir_path,
working_dir=self.workspace_dir)
flow_array = pygeoprocessing.raster_to_numpy_array(
target_flow_dir_path)
self.assertEqual(flow_array.dtype, numpy.uint8)
# this is a regression result saved by hand
expected_result = numpy.array([[2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0],
[4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0],
[4, 4, 2, 2, 2, 2, 2, 2, 2, 0, 0],
[4, 4, 4, 2, 2, 2, 2, 2, 0, 0, 0],
[4, 4, 4, 4, 2, 2, 2, 0, 0, 0, 0],
[4, 4, 4, 4, 4, 2, 0, 0, 0, 0, 0],
[4, 4, 4, 4, 4, 6, 0, 0, 0, 0, 0],
[4, 4, 4, 4, 6, 6, 6, 0, 0, 0, 0],
[4, 4, 4, 6, 6, 6, 6, 6, 0, 0, 0],
[4, 4, 6, 6, 6, 6, 6, 6, 6, 0, 0],
[4, 6, 6, 6, 6, 6, 6, 6, 6, 6, 0]])
numpy.testing.assert_almost_equal(flow_array, expected_result)
def test_flow_dir_mfd_plateau(self):
"""PGP.routing: MFD on a plateau."""
dem_path = os.path.join(self.workspace_dir, 'dem.tif')
# this makes a flat raster
n = 100
dem_array = numpy.zeros((n, n))
dem_nodata = -1
dem_array[2, :] = 1e-12
dem_array[n // 2, :] = 1e-12
dem_array[3 * n // 4, :] = 1e-12
_array_to_raster(dem_array, dem_nodata, dem_path)
target_flow_dir_path = os.path.join(self.workspace_dir, 'flow_dir.tif')
pygeoprocessing.routing.flow_dir_mfd((dem_path, 1),
target_flow_dir_path,
working_dir=self.workspace_dir)
flow_dir_nodata = pygeoprocessing.get_raster_info(
target_flow_dir_path)['nodata'][0]
flow_dir_array = pygeoprocessing.raster_to_numpy_array(
target_flow_dir_path)
self.assertTrue(
not numpy.isclose(flow_dir_array[1:-1, 1:-1],
flow_dir_nodata).any(),
'all flow directions should be defined')
def test_distance_to_channel_d8(self):
"""PGP.routing: test distance to channel D8."""
flow_dir_d8_path = os.path.join(self.workspace_dir, 'flow_dir.d8_tif')
# this is a flow direction raster that was created from a plateau drain
flow_dir_d8_array = numpy.array([[2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0],
[4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0],
[4, 4, 2, 2, 2, 2, 2, 2, 2, 0, 0],
[4, 4, 4, 2, 2, 2, 2, 2, 0, 0, 0],
[4, 4, 4, 4, 2, 2, 2, 0, 0, 0, 0],
[4, 4, 4, 4, 4, 2, 0, 0, 0, 0, 0],
[4, 4, 4, 4, 4, 6, 0, 0, 0, 0, 0],
[4, 4, 4, 4, 6, 6, 6, 0, 0, 0, 0],
[4, 4, 4, 6, 6, 6, 6, 6, 0, 0, 0],
[4, 4, 6, 6, 6, 6, 6, 6, 6, 0, 0],
[4, 6, 6, 6, 6, 6, 6, 6, 6, 6, 0]],
dtype=numpy.uint8)
_array_to_raster(flow_dir_d8_array, None, flow_dir_d8_path)
# taken from a manual inspection of a flow accumulation run
channel_path = os.path.join(self.workspace_dir, 'channel.tif')
channel_array = numpy.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1],
[1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1],
[1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],
dtype=numpy.uint8)
_array_to_raster(channel_array, None, channel_path)
distance_to_channel_d8_path = os.path.join(
self.workspace_dir, 'distance_to_channel_d8.tif')
pygeoprocessing.routing.distance_to_channel_d8(
(flow_dir_d8_path, 1), (channel_path, 1),
distance_to_channel_d8_path)
distance_to_channel_d8_array = pygeoprocessing.raster_to_numpy_array(
distance_to_channel_d8_path)
expected_result = numpy.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
[0, 1, 2, 2, 2, 2, 2, 2, 2, 1, 0],
[0, 1, 2, 3, 3, 3, 3, 3, 2, 1, 0],
[0, 0, 1, 2, 4, 4, 4, 2, 1, 0, 0],
[0, 0, 1, 2, 3, 5, 3, 2, 1, 0, 0],
[0, 0, 1, 2, 3, 4, 3, 2, 1, 0, 0],
[0, 1, 2, 3, 3, 3, 3, 3, 2, 1, 0],
[0, 1, 2, 2, 2, 2, 2, 2, 2, 1, 0],
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
numpy.testing.assert_almost_equal(distance_to_channel_d8_array,
expected_result)
def test_collect_geotiff(self):
"""Datastack: test collect geotiff."""
# Necessary test, as this is proving to be an issue.
from natcap.invest import datastack
params = {
'raster': os.path.join(DATA_DIR, 'landcover.tif'),
}
archive_path = os.path.join(self.workspace, 'archive.invs.tar.gz')
datastack.build_datastack_archive(params, 'sample_model', archive_path)
dest_dir = os.path.join(self.workspace, 'extracted_archive')
archived_params = datastack.extract_datastack_archive(
archive_path, dest_dir)
model_array = pygeoprocessing.raster_to_numpy_array(params['raster'])
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(dest_dir, 'data', archived_params['raster']))
numpy.testing.assert_allclose(model_array, reg_array)
def test_valuation(self):
"""WaveEnergy: testing valuation component."""
from natcap.invest import wave_energy
args = WaveEnergyRegressionTests.generate_base_args(self.workspace_dir)
args['aoi_path'] = os.path.join(SAMPLE_DATA, 'AOI_WCVI.shp')
args['valuation_container'] = True
args['land_gridPts_path'] = os.path.join(SAMPLE_DATA,
'LandGridPts_WCVI.csv')
args['machine_econ_path'] = os.path.join(
SAMPLE_DATA, 'Machine_Pelamis_Economic.csv')
args['number_of_machines'] = 28
# Testing if intermediate/output were overwritten
_make_empty_files(args['workspace_dir'])
wave_energy.execute(args)
raster_results = [
'wp_rc.tif', 'wp_kw.tif', 'capwe_rc.tif', 'capwe_mwh.tif',
'npv_rc.tif', 'npv_usd.tif'
]
for raster_path in raster_results:
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output', raster_path))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, 'valuation', raster_path))
numpy.testing.assert_allclose(model_array, reg_array)
vector_results = ['GridPts_prj.shp', 'LandPts_prj.shp']
for vector_path in vector_results:
WaveEnergyRegressionTests._assert_point_vectors_equal(
os.path.join(args['workspace_dir'], 'output', vector_path),
os.path.join(REGRESSION_DATA, 'valuation', vector_path))
table_results = ['capwe_rc.csv', 'wp_rc.csv', 'npv_rc.csv']
for table_path in table_results:
model_df = pandas.read_csv(
os.path.join(args['workspace_dir'], 'output', table_path))
reg_df = pandas.read_csv(
os.path.join(REGRESSION_DATA, 'valuation', table_path))
pandas.testing.assert_frame_equal(model_df, reg_df)
def test_val_land_grid_points(self):
"""WindEnergy: testing Valuation w/ grid/land pts and wind price."""
from natcap.invest import wind_energy
from natcap.invest.utils import _assert_vectors_equal
args = WindEnergyRegressionTests.generate_base_args(self.workspace_dir)
args['aoi_vector_path'] = os.path.join(SAMPLE_DATA,
'New_England_US_Aoi.shp')
args['land_polygon_vector_path'] = os.path.join(
SAMPLE_DATA, 'simple_north_america_polygon.shp')
args['min_distance'] = 0
args['max_distance'] = 200000
args['valuation_container'] = True
args['foundation_cost'] = 2
args['discount_rate'] = 0.07
# there was no sample data that provided landing points, thus for
# testing, grid points in 'resampled_grid_pts.csv' were duplicated and
# marked as land points. So the distances will be zero, keeping the
# result the same but testing that section of code
args['grid_points_path'] = os.path.join(SAMPLE_DATA,
'resampled_grid_land_pts.csv')
args['price_table'] = False
args['wind_price'] = 0.187
args['rate_change'] = 0.2
wind_energy.execute(args)
raster_results = [
'carbon_emissions_tons.tif', 'levelized_cost_price_per_kWh.tif',
'npv_US_millions.tif'
]
for raster_path in raster_results:
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output', raster_path))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, 'pricevalgridland', raster_path))
# loosened tolerance to pass against GDAL 2.2.4 and 2.4.1
numpy.testing.assert_allclose(model_array, reg_array, rtol=1e-04)
vector_path = 'wind_energy_points.shp'
_assert_vectors_equal(
os.path.join(args['workspace_dir'], 'output', vector_path),
os.path.join(REGRESSION_DATA, 'pricevalgridland', vector_path))
def test_valuation_taskgraph(self):
"""WindEnergy: testing Valuation with async TaskGraph."""
from natcap.invest import wind_energy
from natcap.invest.utils import _assert_vectors_equal
args = WindEnergyRegressionTests.generate_base_args(self.workspace_dir)
# Also use an already projected bathymetry
args['bathymetry_path'] = os.path.join(
SAMPLE_DATA, 'resampled_global_dem_projected.tif')
args['aoi_vector_path'] = os.path.join(SAMPLE_DATA,
'New_England_US_Aoi.shp')
args['land_polygon_vector_path'] = os.path.join(
SAMPLE_DATA, 'simple_north_america_polygon.shp')
args['min_distance'] = 0
args['max_distance'] = 200000
args['valuation_container'] = True
args['foundation_cost'] = 2000000
args['discount_rate'] = 0.07
args['price_table'] = True
args['wind_schedule'] = os.path.join(SAMPLE_DATA,
'price_table_example.csv')
args['wind_price'] = 0.187
args['rate_change'] = 0.2
args['avg_grid_distance'] = 4
args['n_workers'] = 1
wind_energy.execute(args)
raster_results = [
'carbon_emissions_tons.tif', 'levelized_cost_price_per_kWh.tif',
'npv.tif'
]
for raster_path in raster_results:
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output', raster_path))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, 'priceval', raster_path))
numpy.testing.assert_allclose(model_array, reg_array, rtol=1e-6)
vector_path = 'wind_energy_points.shp'
_assert_vectors_equal(
os.path.join(args['workspace_dir'], 'output', vector_path),
os.path.join(REGRESSION_DATA, 'priceval', vector_path))
def test_water_yield_subshed(self):
"""Hydro: testing water yield component only w/ subwatershed."""
from natcap.invest.hydropower import hydropower_water_yield
from natcap.invest import utils
args = HydropowerTests.generate_base_args(self.workspace_dir)
args['sub_watersheds_path'] = os.path.join(SAMPLE_DATA,
'subwatersheds.shp')
args['results_suffix'] = 'test'
hydropower_water_yield.execute(args)
raster_results = ['aet_test.tif', 'fractp_test.tif', 'wyield_test.tif']
for raster_path in raster_results:
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output', 'per_pixel',
raster_path))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, raster_path.replace('_test',
'')))
numpy.testing.assert_allclose(model_array, reg_array, rtol=1e-03)
vector_results = [
'watershed_results_wyield_test.shp',
'subwatershed_results_wyield_test.shp'
]
for vector_path in vector_results:
utils._assert_vectors_equal(
os.path.join(args['workspace_dir'], 'output', vector_path),
os.path.join(REGRESSION_DATA, 'water_yield',
vector_path.replace('_test', '')))
table_results = [
'watershed_results_wyield_test.csv',
'subwatershed_results_wyield_test.csv'
]
for table_path in table_results:
base_table = pandas.read_csv(
os.path.join(args['workspace_dir'], 'output', table_path))
expected_table = pandas.read_csv(
os.path.join(REGRESSION_DATA, 'water_yield',
table_path.replace('_test', '')))
pandas.testing.assert_frame_equal(base_table, expected_table)
def test_valuation_subshed(self):
"""Hydro: testing Valuation component w/ subwatershed."""
from natcap.invest import annual_water_yield
from natcap.invest import utils
args = AnnualWaterYieldTests.generate_base_args(self.workspace_dir)
args['demand_table_path'] = os.path.join(SAMPLE_DATA,
'water_demand_table.csv')
args['valuation_table_path'] = os.path.join(
SAMPLE_DATA, 'hydropower_valuation_table.csv')
args['sub_watersheds_path'] = os.path.join(SAMPLE_DATA,
'subwatersheds.shp')
annual_water_yield.execute(args)
raster_results = ['aet.tif', 'fractp.tif', 'wyield.tif']
for raster_path in raster_results:
model_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output', 'per_pixel',
raster_path))
reg_array = pygeoprocessing.raster_to_numpy_array(
os.path.join(REGRESSION_DATA, raster_path))
numpy.testing.assert_allclose(model_array, reg_array, 1e-03)
vector_results = [
'watershed_results_wyield.shp', 'subwatershed_results_wyield.shp'
]
for vector_path in vector_results:
utils._assert_vectors_equal(
os.path.join(args['workspace_dir'], 'output', vector_path),
os.path.join(REGRESSION_DATA, 'valuation', vector_path))
table_results = [
'watershed_results_wyield.csv', 'subwatershed_results_wyield.csv'
]
for table_path in table_results:
base_table = pandas.read_csv(
os.path.join(args['workspace_dir'], 'output', table_path))
expected_table = pandas.read_csv(
os.path.join(REGRESSION_DATA, 'valuation', table_path))
pandas.testing.assert_frame_equal(base_table, expected_table)
def test_monthly_quickflow_undefined_nodata(self):
"""Test `_calculate_monthly_quick_flow` with undefined nodata values"""
from natcap.invest.seasonal_water_yield import seasonal_water_yield
# set up tiny raster arrays to test
precip_array = numpy.array([[10, 10], [10, 10]], dtype=numpy.float32)
lulc_array = numpy.array([[1, 1], [2, 2]], dtype=numpy.float32)
cn_array = numpy.array([[40, 40], [80, 80]], dtype=numpy.float32)
si_array = numpy.array([[15, 15], [2.5, 2.5]], dtype=numpy.float32)
n_events_array = numpy.array([[10, 10], [1, 1]], dtype=numpy.float32)
stream_mask = numpy.array([[0, 0], [0, 0]], dtype=numpy.float32)
expected_quickflow_array = numpy.array(
[[-4.82284552e-36, -4.82284552e-36],
[6.19275831e-01, 6.19275831e-01]])
precip_path = os.path.join(self.workspace_dir, 'precip.tif')
lulc_path = os.path.join(self.workspace_dir, 'lulc.tif')
cn_path = os.path.join(self.workspace_dir, 'cn.tif')
si_path = os.path.join(self.workspace_dir, 'si.tif')
n_events_path = os.path.join(self.workspace_dir, 'n_events.tif')
stream_path = os.path.join(self.workspace_dir, 'stream.tif')
srs = osr.SpatialReference()
srs.ImportFromEPSG(26910) # UTM Zone 10N
project_wkt = srs.ExportToWkt()
output_path = os.path.join(self.workspace_dir, 'quickflow.tif')
# write all the test arrays to raster files
for array, path in [(precip_array, precip_path),
(lulc_array, lulc_path),
(n_events_array, n_events_path)]:
# make the nodata value undefined for user inputs
pygeoprocessing.numpy_array_to_raster(array, None, (1, -1),
(1180000, 690000),
project_wkt, path)
for array, path in [(cn_array, cn_path), (si_array, si_path),
(stream_mask, stream_path)]:
# define a nodata value for intermediate outputs
pygeoprocessing.numpy_array_to_raster(array, -1, (1, -1),
(1180000, 690000),
project_wkt, path)
# save the quickflow results raster to quickflow.tif
seasonal_water_yield._calculate_monthly_quick_flow(
precip_path, lulc_path, cn_path, n_events_path, stream_path,
si_path, output_path)
# read the raster output back in to a numpy array
quickflow_array = pygeoprocessing.raster_to_numpy_array(output_path)
# assert each element is close to the expected value
self.assertTrue(
numpy.isclose(quickflow_array, expected_quickflow_array).all())
def test_pit_filling(self):
"""PGP.routing: test pitfilling."""
base_path = os.path.join(self.workspace_dir, 'base.tif')
dem_array = numpy.zeros((11, 11), dtype=numpy.float32)
dem_array[3:8, 3:8] = -1.0
dem_array[0, 0] = -1.0
_array_to_raster(dem_array, None, base_path)
fill_path = os.path.join(self.workspace_dir, 'filled.tif')
pygeoprocessing.routing.fill_pits((base_path, 1),
fill_path,
working_dir=self.workspace_dir)
result_array = pygeoprocessing.raster_to_numpy_array(fill_path)
dem_array[3:8, 3:8] = 0.0
numpy.testing.assert_almost_equal(result_array, dem_array)
def test_calculate_distances_land_grid(self):
"""WindEnergy: testing 'calculate_distances_land_grid' function."""
from natcap.invest import wind_energy
srs = osr.SpatialReference()
srs.ImportFromEPSG(3157)
projection_wkt = srs.ExportToWkt()
origin = (443723.127327877911739, 4956546.905980412848294)
pos_x = origin[0]
pos_y = origin[1]
# Setup parameters for creating point shapefile
fields = {'id': ogr.OFTReal, 'L2G': ogr.OFTReal}
attrs = [{'id': 1, 'L2G': 10}, {'id': 2, 'L2G': 20}]
geometries = [
Point(pos_x + 50, pos_y - 50),
Point(pos_x + 50, pos_y - 150)
]
land_shape_path = os.path.join(self.workspace_dir, 'temp_shape.shp')
# Create point shapefile to use for testing input
pygeoprocessing.shapely_geometry_to_vector(geometries,
land_shape_path,
projection_wkt,
'ESRI Shapefile',
fields=fields,
attribute_list=attrs,
ogr_geom_type=ogr.wkbPoint)
# Setup parameters for create raster
matrix = numpy.array([[1, 1, 1, 1], [1, 1, 1, 1]], dtype=numpy.int32)
harvested_masked_path = os.path.join(self.workspace_dir,
'temp_raster.tif')
# Create raster to use for testing input
pygeoprocessing.numpy_array_to_raster(matrix, -1, (100, -100), origin,
projection_wkt,
harvested_masked_path)
tmp_dist_final_path = os.path.join(self.workspace_dir,
'dist_final.tif')
# Call function to test given testing inputs
wind_energy._calculate_distances_land_grid(land_shape_path,
harvested_masked_path,
tmp_dist_final_path, '')
# Compare the results
res_array = pygeoprocessing.raster_to_numpy_array(tmp_dist_final_path)
exp_array = numpy.array([[10, 110, 210, 310], [20, 120, 220, 320]],
dtype=numpy.int32)
numpy.testing.assert_allclose(res_array, exp_array)
def test_create_distance_raster(self):
"""WindEnergy: testing '_create_distance_raster' function."""
from natcap.invest import wind_energy
srs = osr.SpatialReference()
srs.ImportFromEPSG(3157) #UTM Zone 10N
projection_wkt = srs.ExportToWkt()
origin = (443723.127327877911739, 4956546.905980412848294)
pos_x = origin[0]
pos_y = origin[1]
# Setup and create vector to pass to function
fields = {'id': ogr.OFTReal}
attrs = [{'id': 1}]
# Square polygon that will overlap the 4 pixels of the raster in the
# upper left corner
poly_geometry = [box(pos_x, pos_y - 17, pos_x + 17, pos_y)]
poly_vector_path = os.path.join(self.workspace_dir,
'distance_from_vector.gpkg')
# Create polygon shapefile to use as testing input
pygeoprocessing.shapely_geometry_to_vector(
poly_geometry,
poly_vector_path,
projection_wkt,
'GPKG',
fields=fields,
attribute_list=attrs,
ogr_geom_type=ogr.wkbPolygon)
# Create 2x5 raster
matrix = numpy.array([[1, 1, 1, 1, 1], [1, 1, 1, 1, 1]],
dtype=numpy.float32)
base_raster_path = os.path.join(self.workspace_dir, 'temp_raster.tif')
# Create raster to use for testing input
pygeoprocessing.numpy_array_to_raster(matrix, -1, (10, -10), origin,
projection_wkt, base_raster_path)
dist_raster_path = os.path.join(self.workspace_dir, 'dist.tif')
# Call function to test given testing inputs
wind_energy._create_distance_raster(base_raster_path, poly_vector_path,
dist_raster_path,
self.workspace_dir)
# Compare the results
res_array = pygeoprocessing.raster_to_numpy_array(dist_raster_path)
exp_array = numpy.array([[0, 0, 10, 20, 30], [0, 0, 10, 20, 30]],
dtype=numpy.float32)
numpy.testing.assert_allclose(res_array, exp_array)
def test_base_regression(self):
"""SDR base regression test on sample data.
Execute SDR with sample data and checks that the output files are
generated and that the aggregate shapefile fields are the same as the
regression case.
"""
from natcap.invest.sdr import sdr
# use predefined directory so test can clean up files during teardown
args = SDRTests.generate_base_args(self.workspace_dir)
# make args explicit that this is a base run of SWY
sdr.execute(args)
expected_results = {
'usle_tot': 14.25030517578,
'sed_retent': 308382.125,
'sed_export': 0.60502111912,
'sed_dep': 9.05251502991
}
vector_path = os.path.join(args['workspace_dir'],
'watershed_results_sdr.shp')
assert_expected_results_in_vector(expected_results, vector_path)
# We only need to test that the drainage mask exists. Functionality
# for that raster is tested elsewhere
self.assertTrue(
os.path.exists(
os.path.join(args['workspace_dir'], 'intermediate_outputs',
'what_drains_to_stream.tif')))
# Check that sed_deposition does not have any negative, non-nodata
# values, even if they are very small.
sed_deposition_path = os.path.join(args['workspace_dir'],
'sed_deposition.tif')
sed_dep_nodata = pygeoprocessing.get_raster_info(
sed_deposition_path)['nodata'][0]
sed_dep_array = pygeoprocessing.raster_to_numpy_array(
sed_deposition_path)
negative_non_nodata_mask = (
(~numpy.isclose(sed_dep_array, sed_dep_nodata)) &
(sed_dep_array < 0))
self.assertEqual(
numpy.count_nonzero(sed_dep_array[negative_non_nodata_mask]), 0)
def test_lulc_signed_byte(self):
"""Stormwater: regression test for handling signed byte LULC input."""
from natcap.invest import stormwater
(_, biophysical_table_path, _, _, _, soil_group_path, _,
precipitation_path, retention_cost,
pixel_size) = self.basic_setup(self.workspace_dir)
# make custom lulc raster with signed byte type
lulc_array = numpy.array(
[[0, 0, 0, 0], [1, 1, 1, 1], [11, 11, 11, 11], [12, 12, 12, 12]],
dtype=numpy.int8)
lulc_path = os.path.join(self.workspace_dir, 'lulc.tif')
signed_byte_creation_opts = opts_tuple[1] + ('PIXELTYPE=SIGNEDBYTE', )
to_raster(lulc_array,
lulc_path,
raster_driver_creation_tuple=(opts_tuple[0],
signed_byte_creation_opts))
args = {
'workspace_dir': self.workspace_dir,
'results_suffix': '',
'lulc_path': lulc_path,
'soil_group_path': soil_group_path,
'precipitation_path': precipitation_path,
'biophysical_table': biophysical_table_path,
'adjust_retention_ratios': True,
'retention_radius': 20,
'road_centerlines_path': os.path.join(TEST_DATA,
'centerlines.gpkg'),
'aggregate_areas_path': None,
'replacement_cost': retention_cost
}
stormwater.execute(args)
# assert that not all distances to roads are zero
# this problem resulted from not handling signed byte rasters
# when calling `new_raster_from_base`
road_distance_path = os.path.join(self.workspace_dir, 'intermediate',
'road_distance.tif')
distance_is_zero = pygeoprocessing.raster_to_numpy_array(
road_distance_path) == 0
self.assertFalse(numpy.all(distance_is_zero))
def test_flow_accum_d8(self):
"""PGP.routing: test D8 flow accum."""
# this was generated from a pre-calculated plateau drain dem
flow_dir_array = numpy.array([[2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0],
[4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0],
[4, 4, 2, 2, 2, 2, 2, 2, 2, 0, 0],
[4, 4, 4, 2, 2, 2, 2, 2, 0, 0, 0],
[4, 4, 4, 4, 2, 2, 2, 0, 0, 0, 0],
[4, 4, 4, 4, 4, 2, 0, 0, 0, 0, 0],
[4, 4, 4, 4, 4, 6, 0, 0, 0, 0, 0],
[4, 4, 4, 4, 6, 6, 6, 0, 0, 0, 0],
[4, 4, 4, 6, 6, 6, 6, 6, 0, 0, 0],
[4, 4, 6, 6, 6, 6, 6, 6, 6, 0, 0],
[4, 6, 6, 6, 6, 6, 6, 6, 6, 6, 0]],
dtype=numpy.uint8)
flow_dir_path = os.path.join(self.workspace_dir, 'flow_dir.tif')
_array_to_raster(flow_dir_array, None, flow_dir_path)
target_flow_accum_path = os.path.join(self.workspace_dir,
'flow_accum.tif')
pygeoprocessing.routing.flow_accumulation_d8((flow_dir_path, 1),
target_flow_accum_path)
flow_accum_array = pygeoprocessing.raster_to_numpy_array(
target_flow_accum_path)
self.assertEqual(flow_accum_array.dtype, numpy.float64)
# this is a regression result saved by hand
expected_result = numpy.array([[1, 2, 3, 4, 5, 6, 5, 4, 3, 2, 1],
[1, 1, 2, 3, 4, 5, 4, 3, 2, 1, 1],
[2, 1, 1, 2, 3, 4, 3, 2, 1, 1, 2],
[3, 2, 1, 1, 2, 3, 2, 1, 1, 2, 3],
[4, 3, 2, 1, 1, 2, 1, 1, 2, 3, 4],
[5, 4, 3, 2, 1, 1, 1, 2, 3, 4, 5],
[5, 4, 3, 2, 1, 1, 1, 2, 3, 4, 5],
[4, 3, 2, 1, 1, 2, 1, 1, 2, 3, 4],
[3, 2, 1, 1, 2, 3, 2, 1, 1, 2, 3],
[2, 1, 1, 2, 3, 4, 3, 2, 1, 1, 2],
[1, 1, 2, 3, 4, 5, 4, 3, 2, 1, 1]])
numpy.testing.assert_almost_equal(flow_accum_array, expected_result)
def test_pit_filling_nodata_int(self):
"""PGP.routing: test pitfilling with nodata value."""
base_path = os.path.join(self.workspace_dir, 'base.tif')
dem_array = numpy.zeros((11, 11), dtype=numpy.int32)
nodata = 9999
dem_array[3:8, 3:8] = -1
dem_array[0, 0] = -1
dem_array[1, 1] = nodata
_array_to_raster(dem_array, nodata, base_path)
fill_path = os.path.join(self.workspace_dir, 'filled.tif')
pygeoprocessing.routing.fill_pits((base_path, 1),
fill_path,
working_dir=self.workspace_dir)
result_array = pygeoprocessing.raster_to_numpy_array(fill_path)
self.assertEqual(result_array.dtype, numpy.int32)
# the expected result is that the pit is filled in
dem_array[3:8, 3:8] = 0.0
numpy.testing.assert_almost_equal(result_array, dem_array)
def test_extract_streams_mfd(self):
"""PGP.routing: stream extraction on multiple flow direction."""
n = 11
dem_path = os.path.join(self.workspace_dir, 'dem.tif')
dem_array = numpy.zeros((n, n), dtype=numpy.float32)
dem_array[int(n / 2), :] = -1
_array_to_raster(dem_array, None, dem_path)
flow_dir_path = os.path.join(self.workspace_dir, 'flow_dir.tif')
pygeoprocessing.routing.flow_dir_mfd((dem_path, 1), flow_dir_path)
target_flow_accum_path = os.path.join(self.workspace_dir,
'flow_accum_mfd.tif')
pygeoprocessing.routing.flow_accumulation_mfd((flow_dir_path, 1),
target_flow_accum_path)
target_stream_raster_path = os.path.join(self.workspace_dir,
'stream.tif')
pygeoprocessing.routing.extract_streams_mfd(
(target_flow_accum_path, 1), (flow_dir_path, 1),
30,
target_stream_raster_path,
trace_threshold_proportion=0.5)
stream_array = pygeoprocessing.raster_to_numpy_array(
target_stream_raster_path)
expected_stream_array = numpy.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0]])
numpy.testing.assert_almost_equal(stream_array, expected_stream_array)
def test_what_drains_to_stream(self):
"""SDR test for what pixels drain to a stream."""
from natcap.invest.sdr import sdr
srs = osr.SpatialReference()
srs.ImportFromEPSG(26910) # NAD83 / UTM zone 11N
srs_wkt = srs.ExportToWkt()
origin = (463250, 4929700)
pixel_size = (30, -30)
flow_dir_mfd = numpy.array([[0, 1], [1, 1]], dtype=numpy.float64)
flow_dir_mfd_nodata = 0 # Matches pygeoprocessing output
flow_dir_mfd_path = os.path.join(self.workspace_dir, 'flow_dir.tif')
pygeoprocessing.numpy_array_to_raster(flow_dir_mfd,
flow_dir_mfd_nodata, pixel_size,
origin, srs_wkt,
flow_dir_mfd_path)
dist_to_channel = numpy.array([[10, 5], [-1, 6]], dtype=numpy.float64)
dist_to_channel_nodata = -1 # Matches pygeoprocessing output
dist_to_channel_path = os.path.join(self.workspace_dir,
'dist_to_channel.tif')
pygeoprocessing.numpy_array_to_raster(dist_to_channel,
dist_to_channel_nodata,
pixel_size, origin, srs_wkt,
dist_to_channel_path)
target_what_drains_path = os.path.join(self.workspace_dir,
'what_drains.tif')
sdr._calculate_what_drains_to_stream(flow_dir_mfd_path,
dist_to_channel_path,
target_what_drains_path)
# 255 is the byte nodata value assigned
expected_drainage = numpy.array([[255, 1], [0, 1]], dtype=numpy.uint8)
what_drains = pygeoprocessing.raster_to_numpy_array(
target_what_drains_path)
numpy.testing.assert_allclose(what_drains, expected_drainage)
def test_lookup_ratios(self):
"""Stormwater: test lookup_ratios function."""
from natcap.invest import stormwater
sorted_lucodes = [10, 11, 12, 13]
lulc_array = numpy.array([[13, 12], [11, 10]], dtype=numpy.uint8)
soil_group_array = numpy.array([[4, 4], [2, 2]], dtype=numpy.uint8)
lulc_path = os.path.join(self.workspace_dir, 'lulc.tif')
soil_group_path = os.path.join(self.workspace_dir, 'soil_groups.tif')
output_path = os.path.join(self.workspace_dir, 'out.tif')
to_raster(lulc_array, lulc_path, nodata=255)
to_raster(soil_group_array, soil_group_path, nodata=255)
# rows correspond to sorted lucodes, columns to soil groups A-D
ratio_array = numpy.array(
[[0.11, 0.12, 0.13, 0.14], [0.21, 0.22, 0.23, 0.24],
[0.31, 0.32, 0.33, 0.34], [0.41, 0.42, 0.43, 0.44]],
dtype=numpy.float32)
expected_output = numpy.array([[0.44, 0.34], [0.22, 0.12]],
dtype=numpy.float32)
stormwater.lookup_ratios(lulc_path, soil_group_path, ratio_array,
sorted_lucodes, output_path)
actual_output = pygeoprocessing.raster_to_numpy_array(output_path)
numpy.testing.assert_allclose(expected_output, actual_output)
