from wps_tools.testing import run_wps_process, local_path, process_err_test
from quail.processes.wps_climdex_mmdmt import ClimdexMMDMT
from .common import build_file_input
def build_params(climdex_input, month_type, freq, output_file):
return (f"{build_file_input(climdex_input)};"
f"month_type={month_type};"
f"freq={freq};"
f"output_file={output_file};")
@pytest.mark.parametrize(
("climdex_input", "month_type", "freq"),
[
(local_path("climdexInput.rda"), "txx", "monthly"),
(local_path("climdexInput.rda"), "txx", "annual"),
(local_path("climdexInput.rda"), "tnx", "monthly"),
(local_path("climdexInput.rda"), "tnx", "annual"),
(local_path("climdexInput.rds"), "txn", "monthly"),
(local_path("climdexInput.rds"), "txn", "annual"),
(local_path("climdexInput.rds"), "tnn", "monthly"),
(local_path("climdexInput.rds"), "tnn", "annual"),
(
[
local_path("climdexInput.rda"),
local_path("climdexInput.rds"),
local_path("climdex_input_multiple.rda"),
],
"txx",
"monthly",
from wps_tools.testing import run_wps_process, local_path, url_path
from osprey.processes.wps_convert import Convert
from .utils import process_err_test
def build_params(uhs_files, station_file, domain, config_file):
return (f"uhs_files=@xlink:href={uhs_files};"
f"station_file=@xlink:href={station_file};"
f"domain=@xlink:href={domain};"
f"config_file=@xlink:href={config_file};")
@mark.parametrize(
("uhs_files", "station_file", "domain", "config_file"),
[(
local_path("samples/sample.uh_s2"),
local_path("samples/station_file.txt"),
local_path("samples/sample_routing_domain.nc"),
local_path("configs/convert.cfg"),
)],
)
def test_wps_convert_local(uhs_files, station_file, domain, config_file):
params = build_params(uhs_files, station_file, domain, config_file)
run_wps_process(Convert(), params)
@mark.online
@mark.parametrize(
("uhs_files", "station_file", "domain", "config_file"),
[(
local_path("samples/sample.uh_s2"),
"domain",
"input_forcings",
"params_config_file",
"convolve_config_file",
"params_config_dict",
"convolve_config_dict",
),
[
(
"sample",
"COLUMBIA",
"2012-12-01-00",
"2012-12-31",
resource_filename("tests", "data/samples/sample_pour.txt"),
resource_filename("tests", "data/samples/uhbox.csv"),
local_path("samples/sample_flow_parameters.nc"),
local_path("samples/sample_routing_domain.nc"),
url_path("columbia_vicset2.nc", "opendap",
"climate_explorer_data_prep"),
None,
None,
{},
{},
),
],
)
def test_full_rvic_local_pour_points(
case_id,
grid_id,
run_startdate,
stop_date,
from wps_tools.testing import run_wps_process, local_path, process_err_test
from quail.processes.wps_climdex_spells import ClimdexSpells
from .common import build_file_input
def build_params(climdex_input, func, span_years, output_file):
return (f"{build_file_input(climdex_input)}"
f"func={func};"
f"span_years={span_years};"
f"output_file={output_file};")
@pytest.mark.parametrize(
("climdex_input", "func", "span_years"),
[
(local_path("climdexInput.rda"), "wsdi", False),
(local_path("climdexInput.rda"), "wsdi", True),
(local_path("climdexInput.rda"), "csdi", False),
(local_path("climdexInput.rda"), "csdi", True),
(local_path("climdexInput.rds"), "cdd", False),
(local_path("climdexInput.rds"), "cdd", True),
(local_path("climdexInput.rds"), "cwd", False),
(local_path("climdexInput.rds"), "cwd", True),
(local_path("climdexInput.rds"), "wsdi", False),
(
[
local_path("climdexInput.rda"),
local_path("climdexInput.rds"),
local_path("climdex_input_multiple.rda"),
],
"wsdi",
f"data_vector={data_vector};"
f"quantiles_vector={quantiles_vector};"
f"vector_name={vector_name};"
f"output_file={output_file};")
@pytest.mark.parametrize(
(
"data_file",
"data_vector",
"quantiles_vector",
"vector_name",
),
[
(
local_path("ec.1018935.rda"),
"unlist(ec.1018935.tmax['MAX_TEMP'])",
"c(0.1, 0.5, 0.9)",
"tmax_quantiles",
),
(
None,
"0:10",
"0.5",
"quantile_50p",
),
(
local_path("ec.1018935.MAX_TEMP.rds"),
None,
"c(0.1, 0.5, 0.9)",
"tmax_quantiles",
from netCDF4 import Dataset
from tempfile import NamedTemporaryFile
from os import path, remove
NCInput = namedtuple("NCInput", ["url", "file"])
NCInput.__new__.__defaults__ = ("", "")
nc_file = "gdd_annual_CanESM2_rcp85_r1i1p1_1951-2100.nc"
@pytest.mark.online
@pytest.mark.parametrize(
("url"),
[
url_path(nc_file, "opendap"),
local_path(nc_file),
],
)
def test_is_opendap_url(url):
if "docker" in url:
assert is_opendap_url(
url) # Ensure function recognizes this is an opendap file
else:
assert not is_opendap_url(
url) # Ensure function recognizes this is not an opendap file
@pytest.mark.parametrize(
("nc_input"),
[[NCInput(file=local_path(nc_file))]],
)
import pytest
import re
from .utils import TESTDATA
from wps_tools.testing import run_wps_process, local_path, url_path
from thunderbird.processes.wps_generate_prsn import GeneratePrsn
local_nc_inputs = {
nc.split("_")[0]: local_path(nc)
for nc in TESTDATA["test_local_nc"] if nc.startswith("pr_")
or nc.startswith("tasmin_") or nc.startswith("tasmax_")
}
local_tiny_nc_inputs = {
re.sub(".nc", "",
nc.split("_")[-1]): local_path(nc)
for nc in TESTDATA["test_local_nc"] if nc.startswith("tiny_daily")
}
opendap_inputs = {
opendap.split("_")[0]: url_path(opendap, "opendap")
for opendap in TESTDATA["test_opendaps"] if opendap.startswith("pr_")
or opendap.startswith("tasmin_") or opendap.startswith("tasmax_")
}
def build_params(netcdfs, dry_run, chunk_size=None, output_file=None):
output = (f"prec=@xlink:href={netcdfs['pr']};"
f"tasmin=@xlink:href={netcdfs['tasmin']};"
f"tasmax=@xlink:href={netcdfs['tasmax']};"
f"dry_run={dry_run};")
from quail.processes.wps_climdex_rmm import ClimdexRMM
from .common import build_file_input
def build_params(climdex_input, threshold, output_file):
return (
f"{build_file_input(climdex_input)};"
f"threshold={threshold};"
f"output_file={output_file};"
)
@pytest.mark.parametrize(
("climdex_input", "threshold"),
[
(local_path("climdexInput.rda"), 10.0),
(local_path("climdexInput.rda"), 20.0),
(local_path("climdexInput.rds"), 15.5),
(
[
local_path("climdexInput.rda"),
local_path("climdexInput.rds"),
local_path("climdex_input_multiple.rda"),
],
3.0,
),
],
)
def test_wps_climdex_rmm(climdex_input, threshold):
with NamedTemporaryFile(
suffix=".rda", prefix="output_", dir="/tmp", delete=True
from wps_tools.testing import run_wps_process, local_path, process_err_test
from quail.processes.wps_climdex_gsl import ClimdexGSL
from .common import build_file_input
def build_params(climdex_input, gsl_mode, output_file):
return (f"{build_file_input(climdex_input)};"
f"gsl_mode={gsl_mode};"
f"output_file={output_file};")
@pytest.mark.parametrize(
("climdex_input", "gsl_mode"),
[
(
local_path("climdexInput.rda"),
"GSL",
),
(
local_path("climdexInput.rda"),
"GSL_first",
),
(
local_path("climdexInput.rds"),
"GSL_max",
),
(
local_path("climdexInput.rds"),
"GSL_sum",
),
(
import pytest
import os
from pkg_resources import resource_filename, resource_listdir
from .utils import TESTDATA
from wps_tools.testing import run_wps_process, local_path, url_path
from thunderbird.processes.wps_update_metadata import UpdateMetadata
# limiting the test data to tiny_datasets
local_data = [
local_path(nc) for nc in TESTDATA["test_local_nc"] if nc.startswith("tiny_")
]
opendap_data = [
url_path(opendap, "opendap")
for opendap in TESTDATA["test_opendaps"]
if opendap.startswith("tiny_")
]
# updates yaml files
updates_yaml = [
resource_filename("tests", "metadata-conversion/" + test_file)
for test_file in resource_listdir("tests", "metadata-conversion")
]
# updates instruction strings
updates_str = [
"""
global:
history: "today is a nice day"
"""
]
("url"),
[
(url_path("tiny_gcm_climos.nc", "http")),
(url_path("gdd_annual_CanESM2_rcp85_r1i1p1_1951-2100.nc", "http")),
],
)
def test_nc_to_dataset(url):
dataset = nc_to_dataset(url)
assert isinstance(dataset, Dataset)
assert len(dataset.dimensions) > 0
@pytest.mark.parametrize(
("url"),
[local_path("gsl.json")],
)
def test_json_to_dict(url):
output_dict = json_to_dict(url)
assert isinstance(output_dict, dict)
assert len(output_dict) > 0
def txt_to_string_test(url):
output_str = txt_to_string(url)
assert output_str is not None
assert output_str != ""
assert isinstance(output_str, str)
from wps_tools.testing import run_wps_process, local_path, process_err_test
from quail.processes.wps_climdex_temp_pctl import ClimdexTempPctl
from .common import build_file_input
def build_params(climdex_input, func, freq, output_file):
return (f"{build_file_input(climdex_input)}"
f"func={func};"
f"freq={freq};"
f"output_file={output_file};")
@pytest.mark.parametrize(
("climdex_input", "func", "freq"),
[
(local_path("climdexInput.rda"), "tn10p", "monthly"),
(local_path("climdexInput.rda"), "tn10p", "annual"),
(local_path("climdexInput.rda"), "tn90p", "monthly"),
(local_path("climdexInput.rda"), "tn90p", "annual"),
(local_path("climdexInput.rds"), "tx10p", "monthly"),
(local_path("climdexInput.rds"), "tx10p", "annual"),
(local_path("climdexInput.rds"), "tx90p", "monthly"),
(local_path("climdexInput.rds"), "tx90p", "annual"),
(
[
local_path("climdexInput.rda"),
local_path("climdexInput.rds"),
local_path("climdex_input_multiple.rda"),
],
"tn10p",
"monthly",
import pytest
from rpy2 import robjects
from itertools import chain
from tempfile import NamedTemporaryFile
from wps_tools.testing import run_wps_process, local_path, process_err_test
from quail.processes.wps_climdex_get_available_indices import GetIndices
@pytest.mark.parametrize(
("climdex_input", "ci_name"),
[(local_path("climdexInput.rda"), "ci"),
(local_path("climdexInput.rds"), "ci")],
)
def test_wps_get_available_indices_rda(climdex_input, ci_name):
with NamedTemporaryFile(suffix=".rda",
prefix="output_",
dir="/tmp",
delete=True) as out_file:
datainputs = (f"climdex_input=@xlink:href={climdex_input};"
f"ci_name={ci_name};"
f"output_file={out_file.name};")
run_wps_process(GetIndices(), datainputs)
@pytest.mark.parametrize(
("climdex_input", "ci_name"),
[
(local_path("climdexInput.rda"), "not_ci"),
(local_path("expected_dtr.rda"), "expected_dtr_annual"),
],
rule_input = ""
for rule in rules:
rule_input += f"rules={rule};"
return rule_input
@pytest.mark.parametrize(
(
"rules",
"parse_tree",
"variables",
),
[
(
"rule_snow",
local_path("parse_tree.json"),
local_path("collected_variables.json"),
),
(
["rule_snow", "rule_rain", "rule_hybrid"],
local_path("parse_tree.json"),
local_path("collected_variables.json"),
),
],
)
def test_wps_evaluate_rule(rules, parse_tree, variables):
datainputs = (
# rewrite to accept list (see quail)
f"{build_rule_input(rules)}"
f"parse_tree=@xlink:href={parse_tree};"
f"variables=@xlink:href={variables};"
"sample_routing_domain.nc", "opendap", "climate_explorer_data_prep"
),
url_path(
"sample.rvic.prm.COLUMBIA.20180516.nc",
"opendap",
"climate_explorer_data_prep",
),
url_path("columbia_vicset2.nc", "opendap", "climate_explorer_data_prep"),
None,
None,
),
(
"sample",
"2012-12-01-00",
"2012-12-31",
local_path("samples/sample_routing_domain.nc"),
url_path(
"sample.rvic.prm.COLUMBIA.20180516.nc",
"opendap",
"climate_explorer_data_prep",
),
url_path("columbia_vicset2.nc", "opendap", "climate_explorer_data_prep"),
local_path("configs/convolve.cfg"),
None,
),
(
"sample",
"2012-12-01-00",
"2012-12-31",
url_path(
"sample_routing_domain.nc", "opendap", "climate_explorer_data_prep"
from .utils import TESTDATA
from wps_tools.testing import run_wps_process, local_path, url_path, process_err_test
from thunderbird.processes.wps_decompose_flow_vectors import DecomposeFlowVectors
import owslib.wps
import pkg_resources
import os
# limiting test_data to "sample_flow_parameters.nc"
flow_vectors_opendap = [
url_path(opendap, "opendap") for opendap in TESTDATA["test_opendaps"]
if opendap.endswith("sample_flow_parameters.nc")
]
flow_vectors_nc = [
local_path(nc) for nc in TESTDATA["test_local_nc"]
if nc.endswith("sample_flow_parameters.nc")
]
# tiny_datasets do not have flow vectors
non_flow_vectors_nc = [
local_path(nc) for nc in TESTDATA["test_local_nc"]
if nc.startswith("tiny_")
]
def build_params(netcdf, kwargs):
return ("netcdf=@xlink:href={0};"
"dest_file={dest_file};"
"variable={variable};").format(netcdf, **kwargs)
import pytest
from .utils import TESTDATA
from wps_tools.testing import run_wps_process, local_path, url_path, process_err_test
from thunderbird.processes.wps_generate_climos import GenerateClimos
# limiting test_data to non-climo tiny datasets
local_test_data = [
local_path(nc) for nc in TESTDATA["test_local_nc"]
if nc.startswith("tiny_") and not nc.endswith("_climos.nc")
and "_gcm" not in nc
]
# NetCDFs with flow_vectors not adequate for generate_climos
# refer to https://github.com/pacificclimate/climate-explorer-data-prep#generate_climos-generate-climatological-means
opendap_data = [
url_path(opendap, "opendap") for opendap in TESTDATA["test_opendaps"]
if not (opendap.endswith("_climos.nc") or opendap.endswith(
"sample_flow_parameters.nc") or opendap.endswith("19500107.nc"))
]
def build_netcdf(netcdf):
if isinstance(netcdf, str): # Single input file
return f"netcdf=@xlink:href={netcdf};"
else:
nc_input = ""
for nc in netcdf:
nc_input += f"netcdf=@xlink:href={nc};"
return nc_input
def test_local_path(nc_file):
assert f"tests/data/{nc_file}" in local_path(nc_file)
(
"tmax_file",
"tmin_file",
"prec_file",
"tmax_name",
"tmin_name",
"prec_name",
"tmax_column",
"tmin_column",
"prec_column",
"base_range",
"vector_name",
),
[
(
local_path("ec.1018935.rda"),
local_path("ec.1018935.rda"),
local_path("ec.1018935.rda"),
"ec.1018935.tmax",
"ec.1018935.tmin",
"ec.1018935.prec",
"MAX_TEMP",
"MIN_TEMP",
"ONE_DAY_PRECIPITATION",
"c(1971, 2000)",
"climdexInput",
),
(
local_path("ec.1018935.tmax.rds"),
local_path("ec.1018935.tmin.rds"),
local_path("ec.1018935.prec.rds"),
assert test_var[0] == py_var
robjects.r("rm(list=ls())")
@pytest.mark.parametrize(
("name"), [("hello.world"), ("r_name")],
)
def test_r_valid_name(name):
r_valid_name(name)
@pytest.mark.parametrize(
("name"), [(".2"), ("if"), ("two words")],
)
def test_r_valid_name_err(name):
with pytest.raises(ProcessError) as e:
r_valid_name(name)
assert str(vars(e)["_excinfo"][1]) == "Your vector name is not a valid R name"
@pytest.mark.parametrize(
("url"), [(local_path("expected_gsl.rda")), (local_path("expected_days_data.rda"))]
)
def test_get_robjects(url):
objects = get_robjects(url)
assert len(objects) > 0
for ob in objects:
assert isinstance(ob, str)
from wps_tools.testing import run_wps_process, local_path, process_err_test
from quail.processes.wps_climdex_dtr import ClimdexDTR
from .common import build_file_input
def build_params(climdex_input, freq, output_file):
return (f"{build_file_input(climdex_input)}"
f"freq={freq};"
f"output_file={output_file};")
@pytest.mark.parametrize(
("climdex_input", "freq"),
[
(local_path("climdexInput.rda"), "monthly"),
(local_path("climdexInput.rda"), "annual"),
(local_path("climdexInput.rds"), "annual"),
(
[
local_path("climdexInput.rda"),
local_path("climdexInput.rds"),
local_path("climdex_input_multiple.rda"),
],
"monthly",
),
],
)
def test_wps_climdex_dtr(climdex_input, freq):
with NamedTemporaryFile(suffix=".rda",
prefix="output_",
url_path,
)
from wps_tools.testing import run_wps_process
def run_args_collection(wps_test_process_multi_input, file, csv, argc):
params = (";".join([f"file={nc}" for nc in file]) + f';csv_input="{csv}"' +
f";argc={argc};")
run_wps_process(wps_test_process_multi_input, params)
@pytest.mark.parametrize(
("file", "argc"),
[(
[
local_path("tiny_daily_pr.nc"),
local_path("tiny_daily_prsn.nc"),
],
{
"file": 2,
"csv_input": 1,
"argc": 1
},
)],
)
def test_collect_args_local(wps_test_process_multi_input, file, csv_data,
argc):
run_args_collection(wps_test_process_multi_input, file, f"{csv_data}",
argc)
