def test_wps_caps():
client = client_for(Service(processes=processes))
resp = client.get(service='wps',
request='getcapabilities',
version='1.0.0')
names = resp.xpath_text('/wps:Capabilities'
'/wps:ProcessOfferings'
'/wps:Process'
'/ows:Identifier')
assert sorted(names.split()) == [
'bbox',
'binaryoperatorfornumbers',
'chomsky',
'dummyprocess',
'esgf_demo',
'hello',
'inout',
'multiple_outputs',
'nap',
'ncmeta',
'output_formats',
'poly_centroid',
'show_error',
'sleep',
'ultimate_question',
'wordcounter',
]
def test_bbox(self):
if not PY2:
self.skipTest('OWSlib not python 3 compatible')
client = client_for(Service(processes=[create_bbox_process()]))
request_doc = WPS.Execute(OWS.Identifier('my_bbox_process'),
WPS.DataInputs(
WPS.Input(
OWS.Identifier('mybbox'),
WPS.Data(
WPS.BoundingBoxData(
OWS.LowerCorner('15 50'),
OWS.UpperCorner('16 51'),
)))),
version='1.0.0')
resp = client.post_xml(doc=request_doc)
assert_response_success(resp)
[output
] = xpath_ns(resp.xml, '/wps:ExecuteResponse'
'/wps:ProcessOutputs/Output')
self.assertEqual('outbbox',
xpath_ns(output, './ows:Identifier')[0].text)
self.assertEqual(
'15 50',
xpath_ns(output, './ows:BoundingBox/ows:LowerCorner')[0].text)
def test_wps_spatial_analog_process_small_sample():
client = client_for(
Service(processes=[SpatialAnalogProcess()], cfgfiles=CFG_FILE))
datainputs = ("candidate=files@xlink:href={0};"
"target=files@xlink:href={1};"
"location={2},{3};"
"indices={4};"
"indices={5};"
"dist={6};"
"dateStartCandidate={7};"
"dateEndCandidate={8};"
"dateStartTarget={7};"
"dateEndTarget={8}").format(TESTDATA['indicators_small.nc'],
TESTDATA['indicators_medium.nc'],
-72, 46, 'meantemp', 'totalpr',
'kldiv', '1970-01-01',
'1990-01-01')
resp = client.get(service='wps',
request='execute',
version='1.0.0',
identifier='spatial_analog',
datainputs=datainputs)
assert_response_success(resp)
def test_manicouagan(self):
client = client_for(
Service(processes=[HydroBasinsSelectionProcess()],
cfgfiles=CFG_FILE))
fields = [
"location={location}",
# 'level={level}',
# 'lakes={lakes}',
"aggregate_upstream={aggregate_upstream}",
]
datainputs = ";".join(fields).format(
location="-68.724444, 50.646667",
level="12",
lakes=True,
aggregate_upstream=True,
)
resp = client.get(
service="WPS",
request="Execute",
version="1.0.0",
identifier="hydrobasins-select",
datainputs=datainputs,
)
assert_response_success(resp)
out = get_output(resp.xml)
assert {"feature", "upstream_ids"}.issubset([*out])
def test_graph_timeseries_stats(request):
client = client_for(
Service(processes=[GraphIndicatorAnalysis()], cfgfiles=CFG_FILE)
)
datainputs = (
"ts_stats=files@xlink:href=file://{ts_stats};"
"trend={trend};"
"alpha={alpha};".format(
ts_stats=get_local_testdata("ts_stats_outputs/out.nc"),
trend=False,
alpha=0.05,
)
)
resp = client.get(
service="WPS",
request="Execute",
version="1.0.0",
identifier="ts_stats_graph",
datainputs=datainputs,
)
assert_response_success(resp)
out = get_output(resp.xml)
assert out["graph_ts_stats"].endswith(".png")
def describe_process(self, process):
client = client_for(Service(processes=[process]))
resp = client.get(
'?service=wps&version=1.0.0&Request=DescribeProcess&identifier={}'.
format(process.identifier))
[result] = get_describe_result(resp)
return result
def testRegionalisationHMETS(self, method):
client = client_for(
Service(processes=[
RegionalisationProcess(),
], cfgfiles=CFG_FILE))
inp = inputs.copy()
inp['ts'] = TESTDATA['raven-hmets-nc-ts']
inp['model_name'] = 'GR4JCN'
inp['ndonors'] = 2
inp['method'] = method
resp = client.get(service='WPS',
request='execute',
version='1.0.0',
identifier='regionalisation',
datainputs=datainputs.format(**inp))
assert_response_success(resp)
out = get_output(resp.xml)
assert 'hydrograph' in out
hydrograph, _ = urlretrieve(out['hydrograph'])
ensemble, _ = urlretrieve(out['ensemble'])
with xr.open_dataset(hydrograph) as ds:
assert 'q_sim' in ds
with xr.open_dataset(ensemble) as ds:
assert 'realization' in ds.q_sim.dims
def test_wps_caps():
client = client_for(Service(processes=processes))
resp = client.get(service="wps",
request="getcapabilities",
version="1.0.0")
names = resp.xpath_text("/wps:Capabilities"
"/wps:ProcessOfferings"
"/wps:Process"
"/ows:Identifier")
assert sorted(names.split()) == [
"climdex_days",
"climdex_dtr",
"climdex_get_available_indices",
"climdex_gsl",
"climdex_input_csv",
"climdex_input_raw",
"climdex_mmdmt",
"climdex_ptot",
"climdex_quantile",
"climdex_rmm",
"climdex_rxnday",
"climdex_sdii",
"climdex_spells",
"climdex_temp_pctl",
]
def testRegionalisationHMETS(self, method):
client = client_for(
Service(processes=[RegionalisationProcess()], cfgfiles=CFG_FILE))
inp = inputs.copy()
inp["ts"] = get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc",
)
inp["model_name"] = "GR4JCN"
inp["ndonors"] = 2
inp["method"] = method
resp = client.get(
service="WPS",
request="execute",
version="1.0.0",
identifier="regionalisation",
datainputs=datainputs.format(**inp),
)
assert_response_success(resp)
out = get_output(resp.xml)
assert "hydrograph" in out
hydrograph, _ = urlretrieve(out["hydrograph"])
ensemble, _ = urlretrieve(out["ensemble"])
with xr.open_dataset(hydrograph) as ds:
assert "q_sim" in ds
with xr.open_dataset(ensemble) as ds:
assert "realization" in ds.q_sim.dims
def test_hbv_ec(self, tmp_path):
client = client_for(Service(processes=[RavenProcess()], cfgfiles=CFG_FILE))
rvs = get_local_testdata("raven-hbv-ec/raven-hbv-ec-salmon.rv?")
conf = tmp_path / "conf.zip"
with zipfile.ZipFile(conf, "w") as zip:
for rv in rvs:
zip.write(rv, arcname=rv.name)
ts = get_local_testdata("raven-hbv-ec/Salmon-River-Near-Prince-George_*.rvt")
datainputs = (
"ts=files@xlink:href=file://{};"
"ts=files@xlink:href=file://{};"
"conf=files@xlink:href=file://{conf}"
).format(*ts, conf=conf)
resp = client.get(
service="WPS",
request="Execute",
version="1.0.0",
identifier="raven",
datainputs=datainputs,
)
assert_response_success(resp)
def test_wps_empirical_quantile_mapping(netcdf_sdba_ds, kind, name):
client = client_for(
Service(processes=[EmpiricalQuantileMappingProcess()],
cfgfiles=CFG_FILE))
sdba_ds, u = netcdf_sdba_ds
datainputs = (
f"ref=files@xlink:href=file://{sdba_ds[f'qdm_{name}_ref']};"
f"hist=files@xlink:href=file://{sdba_ds[f'qdm_{name}_hist']};"
f"sim=files@xlink:href=file://{sdba_ds[f'qdm_{name}_hist']};"
"group=time;"
f"kind={quote_plus(kind)};"
"nquantiles=50;"
"interp=linear;")
resp = client.get(
f"?service=WPS&request=Execute&version=1.0.0&identifier=empirical_quantile_mapping&datainputs={datainputs}"
)
print(resp.response)
assert_response_success(resp)
out = get_output(resp.xml)
p = xr.open_dataset(out["output"][7:])[name]
uc = convert_calendar(u, "noleap")
middle = ((uc > 1e-2) * (uc < 0.99)).data
ref = xr.open_dataset(sdba_ds[f"qdm_{name}_ref"])[name]
refc = convert_calendar(ref, "noleap")
np.testing.assert_allclose(p[middle], refc[middle], rtol=0.03)
def test_gr4j_salmon_nc(self, tmp_path):
client = client_for(Service(processes=[RavenProcess()], cfgfiles=CFG_FILE))
rvs = get_local_testdata("raven-gr4j-cemaneige/raven-gr4j-salmon.rv?")
conf = tmp_path / "conf.zip"
with zipfile.ZipFile(conf, "w") as zip:
for rv in rvs:
zip.write(rv, arcname=rv.name)
ts = get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
)
datainputs = (
f"ts=files@xlink:href=file://{ts};" f"conf=files@xlink:href=file://{conf}"
)
resp = client.get(
service="WPS",
request="Execute",
version="1.0.0",
identifier="raven",
datainputs=datainputs,
)
assert_response_success(resp)
def test_geoserver_dem_wcs_simple(self):
client = client_for(
Service(processes=[
ZonalStatisticsProcess(),
], cfgfiles=CFG_FILE))
fields = [
'select_all_touching={touches}',
'categorical={categorical}',
'band={band}',
'shape=file@xlink:href=file://{shape}',
]
datainputs = ';'.join(fields).format(
touches=True,
categorical=False,
band=1,
shape=TESTDATA['mrc_subset'],
)
resp = client.get(service='WPS',
request='Execute',
version='1.0.0',
identifier='zonal-stats',
datainputs=datainputs)
assert_response_success(resp)
out = get_output(resp.xml)
feature = json.loads(out['statistics'])['features'][0]
stats = feature['properties']
assert {'count', 'min', 'max', 'mean', 'median', 'sum',
'nodata'}.issubset(stats)
geometry = shape(feature['geometry'])
assert isinstance(type(geometry), type(MultiPolygon))
def test_wps_hello():
client = client_for(Service(processes=[SayHello()]))
datainputs = "name=LovelySugarBird"
resp = client.get(
service='WPS', request='Execute', version='1.0.0', identifier='hello',
datainputs=datainputs)
assert_response_success(resp)
def test_single_polygon(self):
client = client_for(
Service(processes=[TerrainAnalysisProcess()], cfgfiles=CFG_FILE))
fields = [
"shape=file@xlink:href=file://{shape}",
"projected_crs={projected_crs}",
"select_all_touching={touches}",
]
datainputs = ";".join(fields).format(
shape=get_local_testdata("polygons/Basin_10.zip"),
projected_crs="6622",
touches=True,
)
resp = client.get(
service="WPS",
request="Execute",
version="1.0.0",
identifier="terrain-analysis",
datainputs=datainputs,
)
assert_response_success(resp)
out = json.loads(get_output(resp.xml)["properties"])
assert out[0]["elevation"] > 0
assert out[0]["slope"] > 0
assert out[0]["aspect"] > 0
def compare_io(self, name, fn, fmt):
"""Start the dummy process, post the request and check the response matches the input data."""
# Note that `WPSRequest` calls `get_inputs_from_xml` which converts base64 input to bytes
# See `_get_rawvalue_value`
client = client_for(Service(processes=[create_fmt_process(name, fn, fmt)]))
data = get_data(fn, fmt.encoding)
wps = WPSExecution()
doc = wps.buildRequest('test-fmt',
inputs=[('complex', ComplexDataInput(data, mimeType=fmt.mime_type,
encoding=fmt.encoding))],
mode='sync')
resp = client.post_xml(doc=doc)
assert_response_success(resp)
wps.parseResponse(resp.xml)
out = wps.processOutputs[0].data[0]
if 'gml' in fmt.mime_type:
xml_orig = etree.tostring(etree.fromstring(data.encode('utf-8'))).decode('utf-8')
xml_out = etree.tostring(etree.fromstring(out.decode('utf-8'))).decode('utf-8')
# Not equal because the output includes additional namespaces compared to the origin.
# self.assertEqual(xml_out, xml_orig)
else:
self.assertEqual(out.strip(), data.strip())
def test_simple(self):
client = client_for(
Service(processes=[
RasterSubsetProcess(),
], cfgfiles=CFG_FILE))
fields = [
'shape=file@xlink:href=file://{shape}',
'raster=file@xlink:href=file://{raster}',
'band={band}',
'select_all_touching={touches}',
]
datainputs = ';'.join(fields).format(
shape=TESTDATA['mrc_subset'],
raster=TESTDATA['earthenv_dem_90m'],
band=1,
touches=True,
)
resp = client.get(service='WPS',
request='Execute',
version='1.0.0',
identifier='raster-subset',
datainputs=datainputs)
assert_response_success(resp)
out = get_output(resp.xml)
assert {'raster'}.issubset([*out])
def test_thredds():
import lxml.etree
client = client_for(
Service(processes=[SubsetGridPointProcess()], cfgfiles=CFG_FILE))
fn1 = (
"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/cmip5/MRI/rcp85/fx/atmos/r0i0p0/sftlf/"
"sftlf_fx_MRI-CGCM3_rcp85_r0i0p0.nc")
fn2 = (
"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/cmip5/MRI/rcp85/fx/atmos/r0i0p0/orog/"
"orog_fx_MRI-CGCM3_rcp85_r0i0p0.nc")
datainputs = (f"resource=files@xlink:href={fn1};"
f"resource=files@xlink:href={fn2};"
"lat=45.0;"
"lon=150.0;")
resp = client.get(
f"?service=WPS&request=Execute&version=1.0.0&identifier=subset_gridpoint&datainputs={datainputs}"
)
assert_response_success(resp)
out = get_output(resp.xml)
links = get_metalinks(lxml.etree.fromstring(out["ref"].encode()))
assert len(links) == 2
def setUp(self):
def hello(request):
pass
def ping(request):
pass
processes = [
Process(
hello,
'hello',
'Process Hello',
metadata=[
Metadata(
'hello metadata',
'http://example.org/hello',
role=
'http://www.opengis.net/spec/wps/2.0/def/process/description/documentation'
)
]),
Process(ping,
'ping',
'Process Ping',
metadata=[
Metadata('ping metadata', 'http://example.org/ping')
]),
]
self.client = client_for(Service(processes=processes))
def test_ravenpy_error(self):
client = client_for(
Service(processes=[RavenGR4JCemaNeigeProcess()],
cfgfiles=CFG_FILE))
# Deliberate error: no HRUs!
datainputs = (
"ts=files@xlink:href=file://{ts};"
"params={params};"
"start_date={start_date};"
"end_date={end_date};"
"name={name};"
"run_name={run_name};".format(
ts=get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc",
),
params="0.529, -3.396, 407.29, 1.072, 16.9, 0.947",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
name="Salmon",
run_name="test",
))
resp = client.get(
service="WPS",
request="Execute",
version="1.0.0",
identifier="raven-gr4j-cemaneige",
datainputs=datainputs,
)
assert "CHydroUnit constructor:: HRU 1 has a negative or zero area" in str(
resp.data)
def test_notebook(self):
client = client_for(
Service(processes=[
RegionalisationProcess(),
], cfgfiles=CFG_FILE))
inp = inputs.copy()
inp.update(
ts=TESTDATA['raven-hmets-nc-ts'],
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
area=4250.6,
elevation=843.0,
latitude=54.4848,
longitude=-123.3659,
method='PS', # One of the methods described above
model_name=
'HMETS', # One of the two models are allowed: HMETS and GR4JCN
min_nse=0.7,
ndonors=
5, # Number of donors we want to use. Usually between 4 and 8 is a robust number.
properties=json.dumps({
'latitude': 54.4848,
'longitude': -123.3659,
'forest': 0.4
}),
)
resp = client.get(service='WPS',
request='execute',
version='1.0.0',
identifier='regionalisation',
datainputs=datainputs.format(**inp))
assert_response_success(resp)
def test_wps_thredds_workflow():
doc = """
workflow:
name: test_thredds_workflow
source:
thredds:
catalog_url: {0}
worker:
identifier: dummy
url: http://localhost:8091/wps
resource: dataset
inputs: []
""".format(TESTDATA['noaa_catalog_1'])
fp = tempfile.NamedTemporaryFile(suffix=".txt")
yaml.dump(yaml.load(doc), fp)
client = client_for(Service(processes=[DispelWorkflow()]))
datainputs = "workflow@xlink:href=file://{0}".format(fp.name)
resp = client.get(service='wps',
request='execute',
version='1.0.0',
identifier='workflow',
datainputs=datainputs)
print resp.get_data()
assert_response_success(resp)
def create_app():
service = Service(processes=[
Process(say_hello,
inputs=[LiteralInput('name', 'string')],
outputs=[LiteralOutput('response', 'string')]),
Process(feature_count,
inputs=[ComplexInput('layer', [Format(Formats.GML)])],
outputs=[ComplexInput('layer', [Format(Formats.GML)])]),
Process(centroids,
inputs=[ComplexInput('layer', [Format(Formats.GML)])]),
])
app = flask.Flask(__name__)
@app.route('/')
def home():
url = flask.url_for('wps', _external=True)
return flask.render_template('home.html', url=url)
@app.route('/wps', methods=['GET', 'POST'])
def wps():
return service
@app.route('/datafile/<uuid>')
def datafile(uuid):
for data_file in recent_data_files:
if data_file['uuid'] == uuid:
return flask.Response(data_file['bytes'])
else:
flask.abort(404)
return app
def describe_process(self, process):
client = client_for(Service(processes=[process]))
resp = client.get(
'?service=wps&Request=DescribeProcess&identifier=%s' %
process.identifier)
[result] = get_describe_result(resp)
return result
def test_post_with_no_inputs(self):
client = client_for(Service(processes=[create_ultimate_question()]))
request_doc = WPS.Execute(OWS.Identifier('ultimate_question'),
version='1.0.0')
resp = client.post_xml(doc=request_doc)
assert_response_success(resp)
assert get_output(resp.xml) == {'outvalue': '42'}
def test_wps_ultimate_question():
client = client_for(Service(processes=[UltimateQuestion()]))
resp = client.get(service='WPS',
request='Execute',
version='1.0.0',
identifier='ultimate_question')
assert_response_success(resp)
def test_shape_subset(self):
client = client_for(Service(processes=[TerrainAnalysisProcess(), ], cfgfiles=CFG_FILE))
fields = [
'raster=file@xlink:href=file://{raster}',
'shape=file@xlink:href=file://{shape}',
'projected_crs={projected_crs}',
'select_all_touching={touches}',
]
datainputs = ';'.join(fields).format(
raster=TESTDATA['earthenv_dem_90m'],
shape=TESTDATA['mrc_subset'],
projected_crs='6622',
touches=True,
)
resp = client.get(
service='WPS', request='Execute', version='1.0.0', identifier='terrain-analysis', datainputs=datainputs)
assert_response_success(resp)
out = json.loads(get_output(resp.xml)['properties'])
assert out[0]['elevation'] > 0
assert out[0]['slope'] > 0
assert out[0]['aspect'] > 0
def test_shape_subset(self):
client = client_for(
Service(processes=[TerrainAnalysisProcess()], cfgfiles=CFG_FILE))
fields = [
"raster=file@xlink:href=file://{raster}",
"shape=file@xlink:href=file://{shape}",
"projected_crs={projected_crs}",
"select_all_touching={touches}",
]
datainputs = ";".join(fields).format(
raster=get_local_testdata(
"earthenv_dem_90m/earthenv_dem90_southernQuebec.tiff"),
shape=get_local_testdata("donneesqc_mrc_poly/mrc_subset.gml"),
projected_crs="6622",
touches=True,
)
resp = client.get(
service="WPS",
request="Execute",
version="1.0.0",
identifier="terrain-analysis",
datainputs=datainputs,
)
assert_response_success(resp)
out = json.loads(get_output(resp.xml)["properties"])
assert out[0]["elevation"] > 0
assert out[0]["slope"] > 0
assert out[0]["aspect"] > 0
def test_hindcast_graph(self):
client = client_for(
Service(
processes=[
GraphFcstUncertaintyProcess(),
],
cfgfiles=CFG_FILE,
)
)
datainputs = (
"fcst=files@xlink:href=file://{fcst};"
"qobs=files@xlink:href=file://{qobs};"
"fcst_var={fcst_var};"
"qobs_var={qobs_var};".format(
fcst=get_local_testdata("flood_risk/XSS_fcst_ens.nc"),
fcst_var="fcst",
qobs=get_local_testdata("XSS_forecast_data/XSS_obs.nc"),
qobs_var="obs",
)
)
resp = client.get(
service="WPS",
request="Execute",
version="1.0.0",
identifier="graph_forecast_uncertainty",
datainputs=datainputs,
)
assert_response_success(resp)
out = get_output(resp.xml)
assert out["graph_forecasts"].endswith(".png")
def setUp(self):
self.config = ConfigParser.RawConfigParser()
if os.path.isfile('configtests.cfg'):
self.config.read('configtests.cfg')
else:
self.config.read('pavics_datacatalog/tests/configtests.cfg')
try:
self.config_dict = dict(self.config.items('pavicsearch'))
except ConfigParser.NoSectionError:
self.config_dict = {'wps_host': ''}
self.wps_host = self.config_dict['wps_host']
self.solr_host = self.config_dict.get('solr_host', None)
if not self.wps_host:
if self.solr_host:
os.environ['SOLR_HOST'] = self.solr_host
try:
from pavics_datacatalog.wps_processes import \
PavicsSearch
except ImportError:
sys.path.append(
os.path.join('/'.join(os.getcwd().split('/')[:-1]),
'wps_processes'))
from wps_pavicsearch import PavicsSearch
self.client = WpsClient(Service(processes=[PavicsSearch()]),
WpsTestResponse)
else:
self.client = None