from wps_tools.testing import run_wps_process, local_path, process_err_test from quail.processes.wps_climdex_mmdmt import ClimdexMMDMT from .common import build_file_input def build_params(climdex_input, month_type, freq, output_file): return (f"{build_file_input(climdex_input)};" f"month_type={month_type};" f"freq={freq};" f"output_file={output_file};") @pytest.mark.parametrize( ("climdex_input", "month_type", "freq"), [ (local_path("climdexInput.rda"), "txx", "monthly"), (local_path("climdexInput.rda"), "txx", "annual"), (local_path("climdexInput.rda"), "tnx", "monthly"), (local_path("climdexInput.rda"), "tnx", "annual"), (local_path("climdexInput.rds"), "txn", "monthly"), (local_path("climdexInput.rds"), "txn", "annual"), (local_path("climdexInput.rds"), "tnn", "monthly"), (local_path("climdexInput.rds"), "tnn", "annual"), ( [ local_path("climdexInput.rda"), local_path("climdexInput.rds"), local_path("climdex_input_multiple.rda"), ], "txx", "monthly",
from wps_tools.testing import run_wps_process, local_path, url_path from osprey.processes.wps_convert import Convert from .utils import process_err_test def build_params(uhs_files, station_file, domain, config_file): return (f"uhs_files=@xlink:href={uhs_files};" f"station_file=@xlink:href={station_file};" f"domain=@xlink:href={domain};" f"config_file=@xlink:href={config_file};") @mark.parametrize( ("uhs_files", "station_file", "domain", "config_file"), [( local_path("samples/sample.uh_s2"), local_path("samples/station_file.txt"), local_path("samples/sample_routing_domain.nc"), local_path("configs/convert.cfg"), )], ) def test_wps_convert_local(uhs_files, station_file, domain, config_file): params = build_params(uhs_files, station_file, domain, config_file) run_wps_process(Convert(), params) @mark.online @mark.parametrize( ("uhs_files", "station_file", "domain", "config_file"), [( local_path("samples/sample.uh_s2"),
"domain", "input_forcings", "params_config_file", "convolve_config_file", "params_config_dict", "convolve_config_dict", ), [ ( "sample", "COLUMBIA", "2012-12-01-00", "2012-12-31", resource_filename("tests", "data/samples/sample_pour.txt"), resource_filename("tests", "data/samples/uhbox.csv"), local_path("samples/sample_flow_parameters.nc"), local_path("samples/sample_routing_domain.nc"), url_path("columbia_vicset2.nc", "opendap", "climate_explorer_data_prep"), None, None, {}, {}, ), ], ) def test_full_rvic_local_pour_points( case_id, grid_id, run_startdate, stop_date,
from wps_tools.testing import run_wps_process, local_path, process_err_test from quail.processes.wps_climdex_spells import ClimdexSpells from .common import build_file_input def build_params(climdex_input, func, span_years, output_file): return (f"{build_file_input(climdex_input)}" f"func={func};" f"span_years={span_years};" f"output_file={output_file};") @pytest.mark.parametrize( ("climdex_input", "func", "span_years"), [ (local_path("climdexInput.rda"), "wsdi", False), (local_path("climdexInput.rda"), "wsdi", True), (local_path("climdexInput.rda"), "csdi", False), (local_path("climdexInput.rda"), "csdi", True), (local_path("climdexInput.rds"), "cdd", False), (local_path("climdexInput.rds"), "cdd", True), (local_path("climdexInput.rds"), "cwd", False), (local_path("climdexInput.rds"), "cwd", True), (local_path("climdexInput.rds"), "wsdi", False), ( [ local_path("climdexInput.rda"), local_path("climdexInput.rds"), local_path("climdex_input_multiple.rda"), ], "wsdi",
f"data_vector={data_vector};" f"quantiles_vector={quantiles_vector};" f"vector_name={vector_name};" f"output_file={output_file};") @pytest.mark.parametrize( ( "data_file", "data_vector", "quantiles_vector", "vector_name", ), [ ( local_path("ec.1018935.rda"), "unlist(ec.1018935.tmax['MAX_TEMP'])", "c(0.1, 0.5, 0.9)", "tmax_quantiles", ), ( None, "0:10", "0.5", "quantile_50p", ), ( local_path("ec.1018935.MAX_TEMP.rds"), None, "c(0.1, 0.5, 0.9)", "tmax_quantiles",
from netCDF4 import Dataset from tempfile import NamedTemporaryFile from os import path, remove NCInput = namedtuple("NCInput", ["url", "file"]) NCInput.__new__.__defaults__ = ("", "") nc_file = "gdd_annual_CanESM2_rcp85_r1i1p1_1951-2100.nc" @pytest.mark.online @pytest.mark.parametrize( ("url"), [ url_path(nc_file, "opendap"), local_path(nc_file), ], ) def test_is_opendap_url(url): if "docker" in url: assert is_opendap_url( url) # Ensure function recognizes this is an opendap file else: assert not is_opendap_url( url) # Ensure function recognizes this is not an opendap file @pytest.mark.parametrize( ("nc_input"), [[NCInput(file=local_path(nc_file))]], )
import pytest import re from .utils import TESTDATA from wps_tools.testing import run_wps_process, local_path, url_path from thunderbird.processes.wps_generate_prsn import GeneratePrsn local_nc_inputs = { nc.split("_")[0]: local_path(nc) for nc in TESTDATA["test_local_nc"] if nc.startswith("pr_") or nc.startswith("tasmin_") or nc.startswith("tasmax_") } local_tiny_nc_inputs = { re.sub(".nc", "", nc.split("_")[-1]): local_path(nc) for nc in TESTDATA["test_local_nc"] if nc.startswith("tiny_daily") } opendap_inputs = { opendap.split("_")[0]: url_path(opendap, "opendap") for opendap in TESTDATA["test_opendaps"] if opendap.startswith("pr_") or opendap.startswith("tasmin_") or opendap.startswith("tasmax_") } def build_params(netcdfs, dry_run, chunk_size=None, output_file=None): output = (f"prec=@xlink:href={netcdfs['pr']};" f"tasmin=@xlink:href={netcdfs['tasmin']};" f"tasmax=@xlink:href={netcdfs['tasmax']};" f"dry_run={dry_run};")
from quail.processes.wps_climdex_rmm import ClimdexRMM from .common import build_file_input def build_params(climdex_input, threshold, output_file): return ( f"{build_file_input(climdex_input)};" f"threshold={threshold};" f"output_file={output_file};" ) @pytest.mark.parametrize( ("climdex_input", "threshold"), [ (local_path("climdexInput.rda"), 10.0), (local_path("climdexInput.rda"), 20.0), (local_path("climdexInput.rds"), 15.5), ( [ local_path("climdexInput.rda"), local_path("climdexInput.rds"), local_path("climdex_input_multiple.rda"), ], 3.0, ), ], ) def test_wps_climdex_rmm(climdex_input, threshold): with NamedTemporaryFile( suffix=".rda", prefix="output_", dir="/tmp", delete=True
from wps_tools.testing import run_wps_process, local_path, process_err_test from quail.processes.wps_climdex_gsl import ClimdexGSL from .common import build_file_input def build_params(climdex_input, gsl_mode, output_file): return (f"{build_file_input(climdex_input)};" f"gsl_mode={gsl_mode};" f"output_file={output_file};") @pytest.mark.parametrize( ("climdex_input", "gsl_mode"), [ ( local_path("climdexInput.rda"), "GSL", ), ( local_path("climdexInput.rda"), "GSL_first", ), ( local_path("climdexInput.rds"), "GSL_max", ), ( local_path("climdexInput.rds"), "GSL_sum", ), (
import pytest import os from pkg_resources import resource_filename, resource_listdir from .utils import TESTDATA from wps_tools.testing import run_wps_process, local_path, url_path from thunderbird.processes.wps_update_metadata import UpdateMetadata # limiting the test data to tiny_datasets local_data = [ local_path(nc) for nc in TESTDATA["test_local_nc"] if nc.startswith("tiny_") ] opendap_data = [ url_path(opendap, "opendap") for opendap in TESTDATA["test_opendaps"] if opendap.startswith("tiny_") ] # updates yaml files updates_yaml = [ resource_filename("tests", "metadata-conversion/" + test_file) for test_file in resource_listdir("tests", "metadata-conversion") ] # updates instruction strings updates_str = [ """ global: history: "today is a nice day" """ ]
("url"), [ (url_path("tiny_gcm_climos.nc", "http")), (url_path("gdd_annual_CanESM2_rcp85_r1i1p1_1951-2100.nc", "http")), ], ) def test_nc_to_dataset(url): dataset = nc_to_dataset(url) assert isinstance(dataset, Dataset) assert len(dataset.dimensions) > 0 @pytest.mark.parametrize( ("url"), [local_path("gsl.json")], ) def test_json_to_dict(url): output_dict = json_to_dict(url) assert isinstance(output_dict, dict) assert len(output_dict) > 0 def txt_to_string_test(url): output_str = txt_to_string(url) assert output_str is not None assert output_str != "" assert isinstance(output_str, str)
from wps_tools.testing import run_wps_process, local_path, process_err_test from quail.processes.wps_climdex_temp_pctl import ClimdexTempPctl from .common import build_file_input def build_params(climdex_input, func, freq, output_file): return (f"{build_file_input(climdex_input)}" f"func={func};" f"freq={freq};" f"output_file={output_file};") @pytest.mark.parametrize( ("climdex_input", "func", "freq"), [ (local_path("climdexInput.rda"), "tn10p", "monthly"), (local_path("climdexInput.rda"), "tn10p", "annual"), (local_path("climdexInput.rda"), "tn90p", "monthly"), (local_path("climdexInput.rda"), "tn90p", "annual"), (local_path("climdexInput.rds"), "tx10p", "monthly"), (local_path("climdexInput.rds"), "tx10p", "annual"), (local_path("climdexInput.rds"), "tx90p", "monthly"), (local_path("climdexInput.rds"), "tx90p", "annual"), ( [ local_path("climdexInput.rda"), local_path("climdexInput.rds"), local_path("climdex_input_multiple.rda"), ], "tn10p", "monthly",
import pytest from rpy2 import robjects from itertools import chain from tempfile import NamedTemporaryFile from wps_tools.testing import run_wps_process, local_path, process_err_test from quail.processes.wps_climdex_get_available_indices import GetIndices @pytest.mark.parametrize( ("climdex_input", "ci_name"), [(local_path("climdexInput.rda"), "ci"), (local_path("climdexInput.rds"), "ci")], ) def test_wps_get_available_indices_rda(climdex_input, ci_name): with NamedTemporaryFile(suffix=".rda", prefix="output_", dir="/tmp", delete=True) as out_file: datainputs = (f"climdex_input=@xlink:href={climdex_input};" f"ci_name={ci_name};" f"output_file={out_file.name};") run_wps_process(GetIndices(), datainputs) @pytest.mark.parametrize( ("climdex_input", "ci_name"), [ (local_path("climdexInput.rda"), "not_ci"), (local_path("expected_dtr.rda"), "expected_dtr_annual"), ],
rule_input = "" for rule in rules: rule_input += f"rules={rule};" return rule_input @pytest.mark.parametrize( ( "rules", "parse_tree", "variables", ), [ ( "rule_snow", local_path("parse_tree.json"), local_path("collected_variables.json"), ), ( ["rule_snow", "rule_rain", "rule_hybrid"], local_path("parse_tree.json"), local_path("collected_variables.json"), ), ], ) def test_wps_evaluate_rule(rules, parse_tree, variables): datainputs = ( # rewrite to accept list (see quail) f"{build_rule_input(rules)}" f"parse_tree=@xlink:href={parse_tree};" f"variables=@xlink:href={variables};"
"sample_routing_domain.nc", "opendap", "climate_explorer_data_prep" ), url_path( "sample.rvic.prm.COLUMBIA.20180516.nc", "opendap", "climate_explorer_data_prep", ), url_path("columbia_vicset2.nc", "opendap", "climate_explorer_data_prep"), None, None, ), ( "sample", "2012-12-01-00", "2012-12-31", local_path("samples/sample_routing_domain.nc"), url_path( "sample.rvic.prm.COLUMBIA.20180516.nc", "opendap", "climate_explorer_data_prep", ), url_path("columbia_vicset2.nc", "opendap", "climate_explorer_data_prep"), local_path("configs/convolve.cfg"), None, ), ( "sample", "2012-12-01-00", "2012-12-31", url_path( "sample_routing_domain.nc", "opendap", "climate_explorer_data_prep"
from .utils import TESTDATA from wps_tools.testing import run_wps_process, local_path, url_path, process_err_test from thunderbird.processes.wps_decompose_flow_vectors import DecomposeFlowVectors import owslib.wps import pkg_resources import os # limiting test_data to "sample_flow_parameters.nc" flow_vectors_opendap = [ url_path(opendap, "opendap") for opendap in TESTDATA["test_opendaps"] if opendap.endswith("sample_flow_parameters.nc") ] flow_vectors_nc = [ local_path(nc) for nc in TESTDATA["test_local_nc"] if nc.endswith("sample_flow_parameters.nc") ] # tiny_datasets do not have flow vectors non_flow_vectors_nc = [ local_path(nc) for nc in TESTDATA["test_local_nc"] if nc.startswith("tiny_") ] def build_params(netcdf, kwargs): return ("netcdf=@xlink:href={0};" "dest_file={dest_file};" "variable={variable};").format(netcdf, **kwargs)
import pytest from .utils import TESTDATA from wps_tools.testing import run_wps_process, local_path, url_path, process_err_test from thunderbird.processes.wps_generate_climos import GenerateClimos # limiting test_data to non-climo tiny datasets local_test_data = [ local_path(nc) for nc in TESTDATA["test_local_nc"] if nc.startswith("tiny_") and not nc.endswith("_climos.nc") and "_gcm" not in nc ] # NetCDFs with flow_vectors not adequate for generate_climos # refer to https://github.com/pacificclimate/climate-explorer-data-prep#generate_climos-generate-climatological-means opendap_data = [ url_path(opendap, "opendap") for opendap in TESTDATA["test_opendaps"] if not (opendap.endswith("_climos.nc") or opendap.endswith( "sample_flow_parameters.nc") or opendap.endswith("19500107.nc")) ] def build_netcdf(netcdf): if isinstance(netcdf, str): # Single input file return f"netcdf=@xlink:href={netcdf};" else: nc_input = "" for nc in netcdf: nc_input += f"netcdf=@xlink:href={nc};" return nc_input
def test_local_path(nc_file): assert f"tests/data/{nc_file}" in local_path(nc_file)
( "tmax_file", "tmin_file", "prec_file", "tmax_name", "tmin_name", "prec_name", "tmax_column", "tmin_column", "prec_column", "base_range", "vector_name", ), [ ( local_path("ec.1018935.rda"), local_path("ec.1018935.rda"), local_path("ec.1018935.rda"), "ec.1018935.tmax", "ec.1018935.tmin", "ec.1018935.prec", "MAX_TEMP", "MIN_TEMP", "ONE_DAY_PRECIPITATION", "c(1971, 2000)", "climdexInput", ), ( local_path("ec.1018935.tmax.rds"), local_path("ec.1018935.tmin.rds"), local_path("ec.1018935.prec.rds"),
assert test_var[0] == py_var robjects.r("rm(list=ls())") @pytest.mark.parametrize( ("name"), [("hello.world"), ("r_name")], ) def test_r_valid_name(name): r_valid_name(name) @pytest.mark.parametrize( ("name"), [(".2"), ("if"), ("two words")], ) def test_r_valid_name_err(name): with pytest.raises(ProcessError) as e: r_valid_name(name) assert str(vars(e)["_excinfo"][1]) == "Your vector name is not a valid R name" @pytest.mark.parametrize( ("url"), [(local_path("expected_gsl.rda")), (local_path("expected_days_data.rda"))] ) def test_get_robjects(url): objects = get_robjects(url) assert len(objects) > 0 for ob in objects: assert isinstance(ob, str)
from wps_tools.testing import run_wps_process, local_path, process_err_test from quail.processes.wps_climdex_dtr import ClimdexDTR from .common import build_file_input def build_params(climdex_input, freq, output_file): return (f"{build_file_input(climdex_input)}" f"freq={freq};" f"output_file={output_file};") @pytest.mark.parametrize( ("climdex_input", "freq"), [ (local_path("climdexInput.rda"), "monthly"), (local_path("climdexInput.rda"), "annual"), (local_path("climdexInput.rds"), "annual"), ( [ local_path("climdexInput.rda"), local_path("climdexInput.rds"), local_path("climdex_input_multiple.rda"), ], "monthly", ), ], ) def test_wps_climdex_dtr(climdex_input, freq): with NamedTemporaryFile(suffix=".rda", prefix="output_",
url_path, ) from wps_tools.testing import run_wps_process def run_args_collection(wps_test_process_multi_input, file, csv, argc): params = (";".join([f"file={nc}" for nc in file]) + f';csv_input="{csv}"' + f";argc={argc};") run_wps_process(wps_test_process_multi_input, params) @pytest.mark.parametrize( ("file", "argc"), [( [ local_path("tiny_daily_pr.nc"), local_path("tiny_daily_prsn.nc"), ], { "file": 2, "csv_input": 1, "argc": 1 }, )], ) def test_collect_args_local(wps_test_process_multi_input, file, csv_data, argc): run_args_collection(wps_test_process_multi_input, file, f"{csv_data}", argc)