def test_ensemble_heatwave_frequency_grid_point_models_pcic(
mock_datasets, client):
# --- given ---
identifier = "ensemble_grid_point_heat_wave_frequency"
inputs = [
wps_literal_input("lat", "46"),
wps_literal_input("lon", "-72.8"),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("freq", "MS"),
wps_literal_input("models", PCIC_12),
wps_literal_input("output_format", "netcdf"),
]
from pywps.configuration import CONFIG
CONFIG.set("finch", "dataset_bccaqv2", "/mock_local/path")
subset_sample = Path(__file__).parent / "data" / "bccaqv2_subset_sample"
# --- when ---
with mock.patch(
"finch.processes.ensemble_utils.get_bccaqv2_local_files_datasets"
) as mock_datasets:
mock_datasets.return_value = [
str(subset_sample / f) for f in mock_filenames
]
execute_process(client, identifier, inputs)
# --- then ---
assert mock_datasets.call_args[1]["models"] == [PCIC_12]
def test_ensemble_heatwave_frequency_polygon(mock_datasets, client):
# --- given ---
identifier = "ensemble_polygon_heat_wave_frequency"
inputs = [
wps_literal_input("shape", geojson.dumps(poly)),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("thresh_tasmin", "22.0 degC"),
wps_literal_input("thresh_tasmax", "30 degC"),
wps_literal_input("window", "3"),
wps_literal_input("freq", "MS"),
wps_literal_input("ensemble_percentiles", "20, 50, 80"),
wps_literal_input("output_format", "netcdf"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
ds = open_dataset(outputs[0])
dims = dict(ds.dims)
assert dims == {
"lat": 11,
"lon": 11,
"time": 4, # there are roughly 4 months in the test datasets
"rcp": 1
}
data = ds["heat_wave_frequency_p20"].isel(rcp=0, time=1).data
assert np.isnan(data).sum() == 55
assert (~np.isnan(data)).sum() == 66
ensemble_variables = dict(ds.data_vars)
assert sorted(ensemble_variables) == [
f"heat_wave_frequency_p{p}" for p in (20, 50, 80)
]
for var in ensemble_variables.values():
variable_dims = dict(zip(var.dims, var.shape))
assert variable_dims == {"lat": 11, "lon": 11, "time": 4, "rcp": 1}
inputs.append(wps_literal_input("average", "True"))
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
ds = open_dataset(outputs[0])
dims = dict(ds.dims)
assert dims == {"time": 4, "rcp": 1}
ensemble_variables = dict(ds.data_vars)
assert sorted(ensemble_variables) == [
f"heat_wave_frequency_p{p}" for p in (20, 50, 80)
]
for var in ensemble_variables.values():
variable_dims = dict(zip(var.dims, var.shape))
assert variable_dims == {"time": 4, "rcp": 1}
def test_ensemble_heatwave_frequency_bbox(mock_datasets, client):
# --- given ---
identifier = "ensemble_bbox_heat_wave_frequency"
inputs = [
wps_literal_input("lat0", "46.0"),
wps_literal_input("lat1", "46.2"),
wps_literal_input("lon0", "-73.0"),
wps_literal_input("lon1", "-72.8"),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("thresh_tasmin", "22.0 degC"),
wps_literal_input("thresh_tasmax", "30 degC"),
wps_literal_input("window", "3"),
wps_literal_input("freq", "MS"),
wps_literal_input("ensemble_percentiles", "20, 50, 80"),
wps_literal_input("output_format", "netcdf"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
ds = open_dataset(outputs[0])
dims = dict(ds.dims)
assert dims == {
"lat": 2,
"lon": 2,
"time": 4, # there are roughly 4 months in the test datasets
"rcp": 1
}
ensemble_variables = {k: v for k, v in ds.data_vars.items()}
assert sorted(ensemble_variables) == [
f"heat_wave_frequency_p{p}" for p in (20, 50, 80)
]
for var in ensemble_variables.values():
variable_dims = dict(zip(var.dims, var.shape))
assert variable_dims == {"time": 4, "lat": 2, "lon": 2, "rcp": 1}
inputs.append(wps_literal_input("average", "True"))
outputs = execute_process(client, identifier, inputs)
assert len(outputs) == 1
ds = open_dataset(outputs[0])
dims = dict(ds.dims)
assert dims == {"time": 4, "rcp": 1} # Spatial average has been taken.
ensemble_variables = {k: v for k, v in ds.data_vars.items()}
assert sorted(ensemble_variables) == [
f"heat_wave_frequency_p{p}" for p in (20, 50, 80)
]
for var in ensemble_variables.values():
variable_dims = dict(zip(var.dims, var.shape))
assert variable_dims == {"time": 4, "rcp": 1}
def test_ensemble_heatwave_frequency_grid_point_dates(mock_datasets, client):
# --- given ---
identifier = "ensemble_grid_point_heat_wave_frequency"
inputs = [
wps_literal_input("lat", "46"),
wps_literal_input("lon", "-72.8"),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("start_date", "1950"),
wps_literal_input("end_date", "1950-03-31"),
wps_literal_input("freq", "MS"),
wps_literal_input("output_format", "netcdf"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
ds = open_dataset(outputs[0])
dims = dict(ds.dims)
assert dims == {"region": 1, "time": 3, "rcp": 1}
ensemble_variables = dict(ds.data_vars)
assert sorted(ensemble_variables) == [
f"heat_wave_frequency_p{p}" for p in (10, 50, 90)
]
for var in ensemble_variables.values():
variable_dims = dict(zip(var.dims, var.shape))
assert variable_dims == {"region": 1, "time": 3, "rcp": 1}
def test_ensemble_heatwave_frequency_bbox_csv(mock_datasets, client):
# --- given ---
identifier = "ensemble_bbox_heat_wave_frequency"
inputs = [
wps_literal_input("lat0", "46.0"),
wps_literal_input("lat1", "46.2"),
wps_literal_input("lon0", "-73.0"),
wps_literal_input("lon1", "-72.8"),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("thresh_tasmin", "22.0 degC"),
wps_literal_input("thresh_tasmax", "30 degC"),
wps_literal_input("window", "3"),
wps_literal_input("freq", "MS"),
wps_literal_input("ensemble_percentiles", "20, 50, 80"),
wps_literal_input("output_format", "csv"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
zf = zipfile.ZipFile(outputs[0])
assert len(zf.namelist()) == 2 # metadata + data
data_filename = [n for n in zf.namelist() if "metadata" not in n]
csv = zf.read(data_filename[0]).decode()
lines = csv.split("\n")
assert lines[0].startswith("lat,lon,time")
n_data_rows = len(lines) - 2
assert n_data_rows == 2 * 2 * 3 # lat=2, lon=2, time=3 (last month is NaN)
def test_ensemble_compute_intermediate_growing_degree_days_grid_point(
mock_datasets, client):
# --- given ---
identifier = "ensemble_grid_point_growing_degree_days"
inputs = [
wps_literal_input("lat", "46"),
wps_literal_input("lon", "-72.8"),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("ensemble_percentiles", "20, 50, 80"),
wps_literal_input("output_format", "netcdf"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
ds = open_dataset(outputs[0])
dims = dict(ds.dims)
assert dims == {"region": 1, "time": 1, "rcp": 1}
ensemble_variables = dict(ds.data_vars)
assert sorted(ensemble_variables) == [
f"growing_degree_days_p{p}" for p in (20, 50, 80)
]
for var in ensemble_variables.values():
variable_dims = dict(zip(var.dims, var.shape))
assert variable_dims == {"region": 1, "time": 1, "rcp": 1}
def test_wps_averagepoly(client, netcdf_datasets):
# --- given ---
identifier = "average_polygon"
poly = {
"type": "Feature",
"id": "apolygon",
"geometry": {
"type": "Polygon",
"coordinates": [[[0.5, 0], [2.5, 0], [2.5, 2.5], [0.5, 2.5]]],
},
}
inputs = [
wps_input_file("resource", f"file://{netcdf_datasets['tasmin']}"),
wps_literal_input("shape", geojson.dumps(poly)),
wps_literal_input("variable", "tasmin"),
wps_literal_input("start_date", "2000"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
ds = xr.open_dataset(outputs[0])
assert ds.geom.size == 1
assert ds.id.values == ["apolygon"]
def test_tg_mean(client, tas_dataset):
identifier = "tg_mean"
inputs = [wps_input_file("tas", tas_dataset)]
outputs = execute_process(client, identifier, inputs)
ds = xr.open_dataset(outputs[0])
assert ds.tg_mean.standard_name == _get_output_standard_name(identifier)
def test_ensemble_heatwave_frequency_polygon_csv(mock_datasets, client):
# --- given ---
identifier = "ensemble_polygon_heat_wave_frequency"
inputs = [
wps_literal_input("shape", geojson.dumps(poly)),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("thresh_tasmin", "22.0 degC"),
wps_literal_input("thresh_tasmax", "30 degC"),
wps_literal_input("window", "3"),
wps_literal_input("freq", "MS"),
wps_literal_input("ensemble_percentiles", "20, 50, 80"),
wps_literal_input("output_format", "csv"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
zf = zipfile.ZipFile(outputs[0])
assert len(zf.namelist()) == 2 # metadata + data
data_filename = [n for n in zf.namelist() if "metadata" not in n]
csv = zf.read(data_filename[0]).decode()
lines = csv.split("\n")
assert lines[0].startswith("lat,lon,time")
n_data_rows = len(lines) - 2 # header + ending line
# lat-lon=66 (not NaN), time=3 (last month is NaN)
assert n_data_rows == 66 * 3
# --- given ---
inputs.append(wps_literal_input("average", "True"))
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
zf = zipfile.ZipFile(outputs[0])
assert len(zf.namelist()) == 2 # metadata + data
data_filename = [n for n in zf.namelist() if "metadata" not in n]
csv = zf.read(data_filename[0]).decode()
lines = csv.split("\n")
assert lines[0].startswith("time")
n_data_rows = len(lines) - 2 # header + ending line
# after spatial average, time=3 (last month is NaN)
assert n_data_rows == 3
def test_heat_wave_index(client, tasmax_dataset):
identifier = "hwi_{thresh}"
inputs = [
wps_input_file("tasmax", tasmax_dataset),
wps_literal_input("thresh", "30 degC"),
]
outputs = execute_process(client, identifier, inputs)
ds = xr.open_dataset(outputs[0])
assert ds["hwi_30 degC"].standard_name == _get_output_standard_name(
identifier)
def test_heat_wave_frequency(client, tasmin_dataset, tasmax_dataset):
identifier = "heat_wave_frequency"
inputs = [
wps_input_file("tasmax", tasmax_dataset),
wps_input_file("tasmin", tasmin_dataset),
]
outputs = execute_process(client, identifier, inputs)
ds = xr.open_dataset(outputs[0])
assert ds.heat_wave_frequency.standard_name == _get_output_standard_name(
identifier)
def test_bccaqv2_heatwave(mock_compute_indices, mock_bccaq_subset,
mock_datasets, client):
identifier = "BCCAQv2_heat_wave_frequency_gridpoint"
inputs = [
wps_literal_input("output_format", "netcdf"),
wps_literal_input("lat", "2"),
wps_literal_input("lon", "3"),
wps_literal_input("thresh_tasmin", "22.0 degC"),
wps_literal_input("thresh_tasmax", "30 degC"),
wps_literal_input("window", "3"),
wps_literal_input("freq", "YS"),
]
metalink = mock.MagicMock()
tmp = Path(__file__).parent / "tmp"
tmp.mkdir(exist_ok=True)
metalink_file = mock.MagicMock()
metalink_file.file = tmp / "tasmin_some_file.nc"
metalink_file.file.write_text("dummy data")
metalink_file2 = mock.MagicMock()
metalink_file2.file = tmp / "tasmax_some_file.nc"
metalink_file2.file.write_text("dummy data")
metalink.files = [metalink_file, metalink_file2]
mock_datasets.return_value = ["dataset1", "dataset2"]
mock_bccaq_subset.return_value = metalink
def write_dummy_data(filename):
Path(filename).write_text("dummy data")
mock_computed = mock.MagicMock()
mock_compute_indices.return_value = mock_computed
mock_computed.to_netcdf.side_effect = write_dummy_data
outputs = execute_process(client,
identifier,
inputs,
output_names=["output"])
output_file = outputs[0]
assert len(outputs) == 1
assert output_file.endswith("zip")
assert Path(output_file).exists()
assert len(mock_bccaq_subset.call_args[0][0]["resource"]) == 4
def test_wps_subsetpoly(client, netcdf_datasets):
# --- given ---
identifier = "subset_polygon"
inputs = [
wps_input_file("resource", f"file://{netcdf_datasets['tasmin']}"),
wps_literal_input("shape", geojson.dumps(poly)),
wps_literal_input("variable", "tasmin"),
wps_literal_input("start_date", "2000"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
ds = xr.open_dataset(outputs[0])
assert list(ds.lat.values) == [0, 1, 2]
assert list(ds.lon.values) == [1, 2]
def test_bccaqv2_heatwave_online(client):
identifier = "BCCAQv2_heat_wave_frequency_gridpoint"
up_right = 45.507485, -73.541295
inputs = [
wps_literal_input("output_format", "netcdf"),
wps_literal_input("lat", str(up_right[0])),
wps_literal_input("lon", str(up_right[1])),
wps_literal_input("thresh_tasmin", "22.0 degC"),
wps_literal_input("thresh_tasmax", "30 degC"),
wps_literal_input("window", "3"),
wps_literal_input("freq", "YS"),
]
outputs = execute_process(client,
identifier,
inputs,
output_names=["output"])
print(outputs)
def test_wps_averagepoly_shapefile(client, netcdf_datasets, tolerance):
# --- given ---
identifier = "average_polygon"
poly = shapefile_zip()
inputs = [
wps_input_file("resource", f"file://{netcdf_datasets['tasmin']}"),
wps_input_file("shape", poly),
wps_literal_input("tolerance", tolerance),
wps_literal_input("variable", "tasmin"),
wps_literal_input("start_date", "2000"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
ds = xr.open_dataset(outputs[0])
assert ds.geom.size == 1
assert ds.FID.values == [0]
def test_bccaqv2_subset_online(client):
identifier = "subset_ensemble_BCCAQv2"
up_right = 45.507485, -73.541295
bottom_left = 45.385644, -73.691963
inputs = [
wps_literal_input("variable", "tasmin"),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("lon0", str(bottom_left[1])),
wps_literal_input("lon1", str(up_right[1])),
wps_literal_input("lat0", str(bottom_left[0])),
wps_literal_input("lat1", str(up_right[0])),
wps_literal_input("y0", "2010"),
wps_literal_input("y1", "2010"),
]
outputs = execute_process(client,
identifier,
inputs,
output_names=["output"])
print(outputs)
def test_ensemble_heatwave_frequency_grid_point(mock_datasets, client):
# --- given ---
identifier = "ensemble_grid_point_heat_wave_frequency"
inputs = [
wps_literal_input("lat", "46"),
wps_literal_input("lon", "-72.8"),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("thresh_tasmin", "22.0 degC"),
wps_literal_input("thresh_tasmax", "30 degC"),
wps_literal_input("window", "3"),
wps_literal_input("freq", "MS"),
wps_literal_input("ensemble_percentiles", "20, 50, 80"),
wps_literal_input("output_format", "netcdf"),
wps_literal_input("output_name", "testens")
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
assert Path(outputs[0]).stem.startswith('testens')
ds = open_dataset(outputs[0])
dims = dict(ds.dims)
assert dims == {
"region": 1,
"time": 4, # there are roughly 4 months in the test datasets
"rcp": 1
}
ensemble_variables = {k: v for k, v in ds.data_vars.items()}
assert sorted(ensemble_variables) == [
f"heat_wave_frequency_p{p}" for p in (20, 50, 80)
]
for var in ensemble_variables.values():
variable_dims = {d: s for d, s in zip(var.dims, var.shape)}
assert variable_dims == {"region": 1, "time": 4, "rcp": 1}
assert len(ds.attrs['source_datasets'].split('\n')) == 4
def test_bccaqv2_subset(mock_bccaq_subset, mock_datasets, client):
# --- given ---
identifier = "subset_ensemble_BCCAQv2"
inputs = [
wps_literal_input("variable", "tasmin"),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("lat0", "45.507485"),
wps_literal_input("lon0", "-73.541295"),
]
metalink = mock.MagicMock()
metalink_file = mock.MagicMock()
tmp = Path(__file__).parent / "tmp"
metalink_file.file = tmp / "some_file.txt"
if not tmp.exists():
tmp.mkdir()
metalink_file.file.write_text("dummy data")
metalink.files = [metalink_file]
mock_datasets.return_value = ["dataset1", "dataset2"]
mock_bccaq_subset.return_value = metalink
# --- when ---
outputs = execute_process(client,
identifier,
inputs,
output_names=["output"])
# --- then ---
output_file = outputs[0]
assert len(outputs) == 1
assert output_file.endswith("zip")
assert Path(output_file).exists()
data = zipfile.ZipFile(output_file).open("some_file.txt").read()
assert data == b"dummy data"
assert len(mock_bccaq_subset.call_args[0][0]["resource"]) == 2
def test_ensemble_dded_grid_point_multircp(mock_datasets, client):
# --- given ---
identifier = "ensemble_grid_point_degree_days_exceedance_date"
inputs = [
wps_literal_input("lat", "46"),
wps_literal_input("lon", "-72.8"),
wps_literal_input("rcp", "rcp26"),
wps_literal_input("rcp", "rcp45"),
wps_literal_input("thresh", "-5 degC"),
wps_literal_input("sum_thresh", "30 K days"),
wps_literal_input("op", ">"),
wps_literal_input("freq", "MS"),
wps_literal_input("ensemble_percentiles", "20, 50, 80"),
wps_literal_input("output_format", "netcdf"),
]
# --- when ---
outputs = execute_process(client, identifier, inputs)
# --- then ---
assert len(outputs) == 1
ds = open_dataset(outputs[0])
dims = dict(ds.dims)
assert dims == {
"region": 1,
"time": 4, # there are roughly 4 months in the test datasets
"rcp": 2,
}
ensemble_variables = {k: v for k, v in ds.data_vars.items()}
assert sorted(ensemble_variables) == [
f"degree_days_exceedance_date_p{p}" for p in (20, 50, 80)
]
for var in ensemble_variables.values():
variable_dims = dict(zip(var.dims, var.shape))
assert variable_dims == {"region": 1, "time": 4, "rcp": 2}