def test_hindcast_graph(self):
        client = client_for(
            Service(
                processes=[
                    GraphFcstUncertaintyProcess(),
                ],
                cfgfiles=CFG_FILE,
            )
        )

        datainputs = (
            "fcst=files@xlink:href=file://{fcst};"
            "qobs=files@xlink:href=file://{qobs};"
            "fcst_var={fcst_var};"
            "qobs_var={qobs_var};".format(
                fcst=get_local_testdata("flood_risk/XSS_fcst_ens.nc"),
                fcst_var="fcst",
                qobs=get_local_testdata("XSS_forecast_data/XSS_obs.nc"),
                qobs_var="obs",
            )
        )

        resp = client.get(
            service="WPS",
            request="Execute",
            version="1.0.0",
            identifier="graph_forecast_uncertainty",
            datainputs=datainputs,
        )

        assert_response_success(resp)
        out = get_output(resp.xml)
        assert out["graph_forecasts"].endswith(".png")
Example #2
0
def test_freqanalysis_process():
    client = client_for(
        Service(processes=[
            FreqAnalysisProcess(),
        ], cfgfiles=CFG_FILE))

    datainputs = "da=files@xlink:href=file://{da};" \
                 "t={t1};" \
                 "t={t2};" \
                 "dist={dist};" \
                 "freq={freq};" \
                 "mode={mode};" \
                 "season={season};" \
                 "variable={v};" \
        .format(da=TESTDATA['simfile_single'], freq='YS', mode='max', t1=2, t2=50, dist="gumbel_r", season='JJA',
                v='q_sim')

    resp = client.get(service='WPS',
                      request='Execute',
                      version='1.0.0',
                      identifier='freq_analysis',
                      datainputs=datainputs)

    assert_response_success(resp)
    out = get_output(resp.xml)['output']
    ds = xr.open_dataset(out[7:])
    assert ds.freq_analysis.shape == (2, 1)
Example #3
0
    def test_forecast_evaluation_ensemble(self):
        client = client_for(
            Service(processes=[HindcastEvaluationProcess()],
                    cfgfiles=CFG_FILE))

        obs = get_local_testdata("XSS_forecast_data/XSS_obs.nc")
        hcst = get_local_testdata("XSS_forecast_data/XSS_fcst_ens.nc")
        datainputs = (f"obs=files@xlink:href=file://{obs};"
                      f"hcst=files@xlink:href=file://{hcst};"
                      "obs_var=obs;"
                      "hcst_var=fcst;"
                      "metric=crps_ensemble")

        resp = client.get(
            service="WPS",
            request="Execute",
            version="1.0.0",
            identifier="hindcast-evaluation",
            datainputs=datainputs,
        )

        assert_response_success(resp)
        out = get_output(resp.xml)["metrics"]
        m = json.loads(out)
        np.testing.assert_almost_equal(m["crps_ensemble"], 0.1791973, 4)
Example #4
0
def test_wps_spatial_analog_process_small_sample():
    client = client_for(
        Service(processes=[SpatialAnalogProcess()], cfgfiles=CFG_FILE))
    datainputs = ("candidate=files@xlink:href={0};"
                  "target=files@xlink:href={1};"
                  "location={2},{3};"
                  "indices={4};"
                  "indices={5};"
                  "dist={6};"
                  "dateStartCandidate={7};"
                  "dateEndCandidate={8};"
                  "dateStartTarget={7};"
                  "dateEndTarget={8}").format(TESTDATA['indicators_small.nc'],
                                              TESTDATA['indicators_medium.nc'],
                                              -72, 46, 'meantemp', 'totalpr',
                                              'kldiv', '1970-01-01',
                                              '1990-01-01')

    resp = client.get(service='wps',
                      request='execute',
                      version='1.0.0',
                      identifier='spatial_analog',
                      datainputs=datainputs)

    assert_response_success(resp)
Example #5
0
    def test_gr4j_salmon_nc(self, tmp_path):
        client = client_for(Service(processes=[RavenProcess()], cfgfiles=CFG_FILE))

        rvs = get_local_testdata("raven-gr4j-cemaneige/raven-gr4j-salmon.rv?")
        conf = tmp_path / "conf.zip"
        with zipfile.ZipFile(conf, "w") as zip:
            for rv in rvs:
                zip.write(rv, arcname=rv.name)

        ts = get_local_testdata(
            "raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
        )
        datainputs = (
            f"ts=files@xlink:href=file://{ts};" f"conf=files@xlink:href=file://{conf}"
        )

        resp = client.get(
            service="WPS",
            request="Execute",
            version="1.0.0",
            identifier="raven",
            datainputs=datainputs,
        )

        assert_response_success(resp)
Example #6
0
    def test_bbox(self):
        if not PY2:
            self.skipTest('OWSlib not python 3 compatible')
        client = client_for(Service(processes=[create_bbox_process()]))
        request_doc = WPS.Execute(
            OWS.Identifier('my_bbox_process'),
            WPS.DataInputs(
                WPS.Input(
                    OWS.Identifier('mybbox'),
                    WPS.Data(WPS.BoundingBoxData(
                        OWS.LowerCorner('15 50'),
                        OWS.UpperCorner('16 51'),
                    ))
                )
            ),
            version='1.0.0'
        )
        resp = client.post_xml(doc=request_doc)
        assert_response_success(resp)

        [output] = xpath_ns(resp.xml, '/wps:ExecuteResponse'
                                      '/wps:ProcessOutputs/wps:Output')
        self.assertEqual('outbbox', xpath_ns(
            output,
            './ows:Identifier')[0].text)
        self.assertEqual('15 50', xpath_ns(
            output,
            './wps:Data/ows:BoundingBox/ows:LowerCorner')[0].text)
    def test_geoserver_dem_wcs_simple(self):
        client = client_for(
            Service(processes=[
                ZonalStatisticsProcess(),
            ], cfgfiles=CFG_FILE))
        fields = [
            'select_all_touching={touches}',
            'categorical={categorical}',
            'band={band}',
            'shape=file@xlink:href=file://{shape}',
        ]

        datainputs = ';'.join(fields).format(
            touches=True,
            categorical=False,
            band=1,
            shape=TESTDATA['mrc_subset'],
        )
        resp = client.get(service='WPS',
                          request='Execute',
                          version='1.0.0',
                          identifier='zonal-stats',
                          datainputs=datainputs)

        assert_response_success(resp)
        out = get_output(resp.xml)
        feature = json.loads(out['statistics'])['features'][0]

        stats = feature['properties']
        assert {'count', 'min', 'max', 'mean', 'median', 'sum',
                'nodata'}.issubset(stats)

        geometry = shape(feature['geometry'])
        assert isinstance(type(geometry), type(MultiPolygon))
Example #8
0
    def test_simple(self):
        client = client_for(
            Service(processes=[
                RasterSubsetProcess(),
            ], cfgfiles=CFG_FILE))

        fields = [
            'shape=file@xlink:href=file://{shape}',
            'raster=file@xlink:href=file://{raster}',
            'band={band}',
            'select_all_touching={touches}',
        ]

        datainputs = ';'.join(fields).format(
            shape=TESTDATA['mrc_subset'],
            raster=TESTDATA['earthenv_dem_90m'],
            band=1,
            touches=True,
        )

        resp = client.get(service='WPS',
                          request='Execute',
                          version='1.0.0',
                          identifier='raster-subset',
                          datainputs=datainputs)

        assert_response_success(resp)
        out = get_output(resp.xml)

        assert {'raster'}.issubset([*out])
    def test_manicouagan(self):
        client = client_for(
            Service(processes=[HydroBasinsSelectionProcess()],
                    cfgfiles=CFG_FILE))

        fields = [
            "location={location}",
            # 'level={level}',
            # 'lakes={lakes}',
            "aggregate_upstream={aggregate_upstream}",
        ]

        datainputs = ";".join(fields).format(
            location="-68.724444, 50.646667",
            level="12",
            lakes=True,
            aggregate_upstream=True,
        )

        resp = client.get(
            service="WPS",
            request="Execute",
            version="1.0.0",
            identifier="hydrobasins-select",
            datainputs=datainputs,
        )

        assert_response_success(resp)
        out = get_output(resp.xml)

        assert {"feature", "upstream_ids"}.issubset([*out])
    def test_notebook(self):
        client = client_for(
            Service(processes=[
                RegionalisationProcess(),
            ], cfgfiles=CFG_FILE))

        inp = inputs.copy()
        inp.update(
            ts=TESTDATA['raven-hmets-nc-ts'],
            start_date=dt.datetime(2000, 1, 1),
            end_date=dt.datetime(2002, 1, 1),
            area=4250.6,
            elevation=843.0,
            latitude=54.4848,
            longitude=-123.3659,
            method='PS',  # One of the methods described above
            model_name=
            'HMETS',  # One of the two models are allowed: HMETS and GR4JCN
            min_nse=0.7,
            ndonors=
            5,  # Number of donors we want to use. Usually between 4 and 8 is a robust number.
            properties=json.dumps({
                'latitude': 54.4848,
                'longitude': -123.3659,
                'forest': 0.4
            }),
        )

        resp = client.get(service='WPS',
                          request='execute',
                          version='1.0.0',
                          identifier='regionalisation',
                          datainputs=datainputs.format(**inp))

        assert_response_success(resp)
    def test_single_polygon(self):
        client = client_for(
            Service(processes=[TerrainAnalysisProcess()], cfgfiles=CFG_FILE))
        fields = [
            "shape=file@xlink:href=file://{shape}",
            "projected_crs={projected_crs}",
            "select_all_touching={touches}",
        ]

        datainputs = ";".join(fields).format(
            shape=get_local_testdata("polygons/Basin_10.zip"),
            projected_crs="6622",
            touches=True,
        )

        resp = client.get(
            service="WPS",
            request="Execute",
            version="1.0.0",
            identifier="terrain-analysis",
            datainputs=datainputs,
        )

        assert_response_success(resp)
        out = json.loads(get_output(resp.xml)["properties"])

        assert out[0]["elevation"] > 0
        assert out[0]["slope"] > 0
        assert out[0]["aspect"] > 0
def test_thredds():
    import lxml.etree

    client = client_for(
        Service(processes=[SubsetGridPointProcess()], cfgfiles=CFG_FILE))
    fn1 = (
        "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/cmip5/MRI/rcp85/fx/atmos/r0i0p0/sftlf/"
        "sftlf_fx_MRI-CGCM3_rcp85_r0i0p0.nc")
    fn2 = (
        "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/cmip5/MRI/rcp85/fx/atmos/r0i0p0/orog/"
        "orog_fx_MRI-CGCM3_rcp85_r0i0p0.nc")

    datainputs = (f"resource=files@xlink:href={fn1};"
                  f"resource=files@xlink:href={fn2};"
                  "lat=45.0;"
                  "lon=150.0;")

    resp = client.get(
        f"?service=WPS&request=Execute&version=1.0.0&identifier=subset_gridpoint&datainputs={datainputs}"
    )

    assert_response_success(resp)
    out = get_output(resp.xml)
    links = get_metalinks(lxml.etree.fromstring(out["ref"].encode()))
    assert len(links) == 2
    def test_forecast_floodrisk_ensemble(self):
        client = client_for(
            Service(
                processes=[
                    ForecastFloodRiskProcess(),
                ],
                cfgfiles=CFG_FILE,
            )
        )

        datainputs = (
            "fcst=files@xlink:href=file://{fcst};"
            "name=fcst;"
            "flood_level={flood_level};".format(
                fcst=get_local_testdata("flood_risk/XSS_fcst_ens.nc"), flood_level=0.5
            )
        )

        resp = client.get(
            service="WPS",
            request="Execute",
            version="1.0.0",
            identifier="forecast-floodrisk",
            datainputs=datainputs,
        )

        assert_response_success(resp)
        out = get_output(resp.xml)
        floodrisk, _ = urlretrieve(out["flood_risk"])
        ds = xr.open_dataset(floodrisk)
        np.testing.assert_almost_equal(ds["fcst"], [0.2, 0.2, 0.8], 2)
Example #14
0
    def testRegionalisationHMETS(self, method):
        client = client_for(
            Service(processes=[RegionalisationProcess()], cfgfiles=CFG_FILE))

        inp = inputs.copy()
        inp["ts"] = get_local_testdata(
            "raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc",
        )
        inp["model_name"] = "GR4JCN"
        inp["ndonors"] = 2
        inp["method"] = method

        resp = client.get(
            service="WPS",
            request="execute",
            version="1.0.0",
            identifier="regionalisation",
            datainputs=datainputs.format(**inp),
        )

        assert_response_success(resp)
        out = get_output(resp.xml)
        assert "hydrograph" in out
        hydrograph, _ = urlretrieve(out["hydrograph"])
        ensemble, _ = urlretrieve(out["ensemble"])

        with xr.open_dataset(hydrograph) as ds:
            assert "q_sim" in ds

        with xr.open_dataset(ensemble) as ds:
            assert "realization" in ds.q_sim.dims
Example #15
0
def test_graph_timeseries_stats(request):
    client = client_for(
        Service(processes=[GraphIndicatorAnalysis()], cfgfiles=CFG_FILE)
    )

    datainputs = (
        "ts_stats=files@xlink:href=file://{ts_stats};"
        "trend={trend};"
        "alpha={alpha};".format(
            ts_stats=get_local_testdata("ts_stats_outputs/out.nc"),
            trend=False,
            alpha=0.05,
        )
    )

    resp = client.get(
        service="WPS",
        request="Execute",
        version="1.0.0",
        identifier="ts_stats_graph",
        datainputs=datainputs,
    )

    assert_response_success(resp)
    out = get_output(resp.xml)
    assert out["graph_ts_stats"].endswith(".png")
    def test_rmse_gr4j(self, gr4j):
        client = client_for(
            Service(
                processes=[
                    ObjectiveFunctionProcess(),
                ],
                cfgfiles=CFG_FILE,
            )
        )

        kwds = dict(hydrograph=gr4j.outputs["hydrograph"], name="rmse")

        datainputs = (
            "obs=files@xlink:href=file://{hydrograph};"
            "sim=files@xlink:href=file://{hydrograph};"
            "name={name}".format(**kwds)
        )

        resp = client.get(
            service="WPS",
            request="Execute",
            version="1.0.0",
            identifier="objective-function",
            datainputs=datainputs,
        )

        assert_response_success(resp)
        out = get_output(resp.xml)["metrics"]
        m = json.loads(out)

        np.testing.assert_almost_equal(m["rmse"], gr4j.diagnostics["DIAG_RMSE"], 4)
    def test_shape_subset(self):
        client = client_for(Service(processes=[TerrainAnalysisProcess(), ], cfgfiles=CFG_FILE))
        fields = [
            'raster=file@xlink:href=file://{raster}',
            'shape=file@xlink:href=file://{shape}',
            'projected_crs={projected_crs}',
            'select_all_touching={touches}',
        ]

        datainputs = ';'.join(fields).format(
            raster=TESTDATA['earthenv_dem_90m'],
            shape=TESTDATA['mrc_subset'],
            projected_crs='6622',
            touches=True,
        )

        resp = client.get(
            service='WPS', request='Execute', version='1.0.0', identifier='terrain-analysis', datainputs=datainputs)

        assert_response_success(resp)
        out = json.loads(get_output(resp.xml)['properties'])

        assert out[0]['elevation'] > 0
        assert out[0]['slope'] > 0
        assert out[0]['aspect'] > 0
Example #18
0
    def test_smallwood_reservoir(self):
        client = client_for(
            Service(processes=[
                HydroBasinsSelectionProcess(),
            ],
                    cfgfiles=CFG_FILE))

        fields = [
            'location={location}',
            # 'level={level}',
            # 'lakes={lakes}',
            'aggregate_upstream={aggregate_upstream}',
        ]

        datainputs = ';'.join(fields).format(
            location="-64.2, 54.1",
            level="12",
            lakes=True,
            aggregate_upstream=True,
        )

        resp = client.get(service='WPS',
                          request='Execute',
                          version='1.0.0',
                          identifier='hydrobasins-select',
                          datainputs=datainputs)

        assert_response_success(resp)
        out = get_output(resp.xml)

        assert {'feature', 'upstream_ids'}.issubset([*out])
    def testRegionalisationHMETS(self, method):
        client = client_for(
            Service(processes=[
                RegionalisationProcess(),
            ], cfgfiles=CFG_FILE))

        inp = inputs.copy()
        inp['ts'] = TESTDATA['raven-hmets-nc-ts']
        inp['model_name'] = 'GR4JCN'
        inp['ndonors'] = 2
        inp['method'] = method

        resp = client.get(service='WPS',
                          request='execute',
                          version='1.0.0',
                          identifier='regionalisation',
                          datainputs=datainputs.format(**inp))

        assert_response_success(resp)
        out = get_output(resp.xml)
        assert 'hydrograph' in out
        hydrograph, _ = urlretrieve(out['hydrograph'])
        ensemble, _ = urlretrieve(out['ensemble'])

        with xr.open_dataset(hydrograph) as ds:
            assert 'q_sim' in ds

        with xr.open_dataset(ensemble) as ds:
            assert 'realization' in ds.q_sim.dims
Example #20
0
    def test_hbv_ec(self, tmp_path):
        client = client_for(Service(processes=[RavenProcess()], cfgfiles=CFG_FILE))

        rvs = get_local_testdata("raven-hbv-ec/raven-hbv-ec-salmon.rv?")
        conf = tmp_path / "conf.zip"
        with zipfile.ZipFile(conf, "w") as zip:
            for rv in rvs:
                zip.write(rv, arcname=rv.name)

        ts = get_local_testdata("raven-hbv-ec/Salmon-River-Near-Prince-George_*.rvt")

        datainputs = (
            "ts=files@xlink:href=file://{};"
            "ts=files@xlink:href=file://{};"
            "conf=files@xlink:href=file://{conf}"
        ).format(*ts, conf=conf)

        resp = client.get(
            service="WPS",
            request="Execute",
            version="1.0.0",
            identifier="raven",
            datainputs=datainputs,
        )
        assert_response_success(resp)
Example #21
0
def test_wps_subset_c3s_cmip6_metadata():
    client = client_for(Service(processes=[Subset()], cfgfiles=[PYWPS_CFG]))
    datainputs = f"collection={C3S_CMIP6_MON_TASMIN_COLLECTION}"
    datainputs += ";time=2010/2010"
    resp = client.get(
        f"?service=WPS&request=Execute&version=1.0.0&identifier=subset&lineage=true&datainputs={datainputs}"
    )
    # print(resp.data)
    assert_response_success(resp)
    m_path = get_output(resp.xml)["output"]
    assert "meta4" in m_path
    # parse metalink
    xml = open(m_path[7:], "r").read()
    urls = parse_metalink(xml)
    # print(urls)
    ds = xr.open_dataset(urls[0][7:], use_cftime=True)
    # check fill value in bounds
    assert "_FillValue" not in ds.lat_bnds.encoding
    assert "_FillValue" not in ds.lon_bnds.encoding
    assert "_FillValue" not in ds.time_bnds.encoding
    # check fill value in coordinates
    assert "_FillValue" not in ds.time.encoding
    assert "_FillValue" not in ds.lat.encoding
    assert "_FillValue" not in ds.lon.encoding
    assert "_FillValue" not in ds.height.encoding
    # check coordinates in bounds
    assert "coordinates" not in ds.lat_bnds.encoding
    assert "coordinates" not in ds.lon_bnds.encoding
    assert "coordinates" not in ds.time_bnds.encoding
Example #22
0
 def test_post_with_no_inputs(self):
     client = client_for(Service(processes=[create_ultimate_question()]))
     request_doc = WPS.Execute(OWS.Identifier('ultimate_question'),
                               version='1.0.0')
     resp = client.post_xml(doc=request_doc)
     assert_response_success(resp)
     assert get_output(resp.xml) == {'outvalue': '42'}
Example #23
0
def test_wps_ultimate_question():
    client = client_for(Service(processes=[UltimateQuestion()]))
    resp = client.get(service='WPS',
                      request='Execute',
                      version='1.0.0',
                      identifier='ultimate_question')
    assert_response_success(resp)
Example #24
0
    def compare_io(self, name, fn, fmt):
        """Start the dummy process, post the request and check the response matches the input data."""

        # Note that `WPSRequest` calls `get_inputs_from_xml` which converts base64 input to bytes
        # See `_get_rawvalue_value`
        client = client_for(Service(processes=[create_fmt_process(name, fn, fmt)]))
        data = get_data(fn, fmt.encoding)

        wps = WPSExecution()
        doc = wps.buildRequest('test-fmt',
                               inputs=[('complex', ComplexDataInput(data, mimeType=fmt.mime_type,
                                                                    encoding=fmt.encoding))],
                               mode='sync')
        resp = client.post_xml(doc=doc)
        assert_response_success(resp)
        wps.parseResponse(resp.xml)
        out = wps.processOutputs[0].data[0]

        if 'gml' in fmt.mime_type:
            xml_orig = etree.tostring(etree.fromstring(data.encode('utf-8'))).decode('utf-8')
            xml_out = etree.tostring(etree.fromstring(out.decode('utf-8'))).decode('utf-8')
            # Not equal because the output includes additional namespaces compared to the origin.
            # self.assertEqual(xml_out, xml_orig)

        else:
            self.assertEqual(out.strip(), data.strip())
Example #25
0
def test_wps_hello():
    client = client_for(Service(processes=[SayHello()]))
    datainputs = "name=LovelySugarBird"
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0', identifier='hello',
        datainputs=datainputs)
    assert_response_success(resp)
    def test_shape_subset(self):
        client = client_for(
            Service(processes=[TerrainAnalysisProcess()], cfgfiles=CFG_FILE))
        fields = [
            "raster=file@xlink:href=file://{raster}",
            "shape=file@xlink:href=file://{shape}",
            "projected_crs={projected_crs}",
            "select_all_touching={touches}",
        ]

        datainputs = ";".join(fields).format(
            raster=get_local_testdata(
                "earthenv_dem_90m/earthenv_dem90_southernQuebec.tiff"),
            shape=get_local_testdata("donneesqc_mrc_poly/mrc_subset.gml"),
            projected_crs="6622",
            touches=True,
        )

        resp = client.get(
            service="WPS",
            request="Execute",
            version="1.0.0",
            identifier="terrain-analysis",
            datainputs=datainputs,
        )

        assert_response_success(resp)
        out = json.loads(get_output(resp.xml)["properties"])

        assert out[0]["elevation"] > 0
        assert out[0]["slope"] > 0
        assert out[0]["aspect"] > 0
Example #27
0
    def test_bbox(self):
        client = client_for(Service(processes=[create_bbox_process()]))
        request_doc = WPS.Execute(OWS.Identifier('my_bbox_process'),
                                  WPS.DataInputs(
                                      WPS.Input(
                                          OWS.Identifier('mybbox'),
                                          WPS.Data(
                                              WPS.BoundingBoxData(
                                                  OWS.LowerCorner('15 50'),
                                                  OWS.UpperCorner('16 51'),
                                              )))),
                                  version='1.0.0')
        resp = client.post_xml(doc=request_doc)
        assert_response_success(resp)

        [output
         ] = xpath_ns(resp.xml, '/wps:ExecuteResponse'
                      '/wps:ProcessOutputs/wps:Output')
        self.assertEqual('outbbox',
                         xpath_ns(output, './ows:Identifier')[0].text)

        self.assertEqual(
            ' 15  50 ',
            xpath_ns(
                output,
                './wps:Data/ows:WGS84BoundingBox/ows:LowerCorner')[0].text)

        self.assertEqual(
            ' 16  51 ',
            xpath_ns(
                output,
                './wps:Data/ows:WGS84BoundingBox/ows:UpperCorner')[0].text)
def test_wps_thredds_workflow():
    doc = """
    workflow:
      name: test_thredds_workflow
    source:
      thredds:
        catalog_url: {0}
    worker:
      identifier: dummy
      url: http://localhost:8091/wps
      resource: dataset
      inputs: []
    """.format(TESTDATA['noaa_catalog_1'])
    fp = tempfile.NamedTemporaryFile(suffix=".txt")
    yaml.dump(yaml.load(doc), fp)

    client = client_for(Service(processes=[DispelWorkflow()]))
    datainputs = "workflow@xlink:href=file://{0}".format(fp.name)
    resp = client.get(service='wps',
                      request='execute',
                      version='1.0.0',
                      identifier='workflow',
                      datainputs=datainputs)
    print resp.get_data()
    assert_response_success(resp)
Example #29
0
def test_wps_empirical_quantile_mapping(netcdf_sdba_ds, kind, name):
    client = client_for(
        Service(processes=[EmpiricalQuantileMappingProcess()],
                cfgfiles=CFG_FILE))

    sdba_ds, u = netcdf_sdba_ds

    datainputs = (
        f"ref=files@xlink:href=file://{sdba_ds[f'qdm_{name}_ref']};"
        f"hist=files@xlink:href=file://{sdba_ds[f'qdm_{name}_hist']};"
        f"sim=files@xlink:href=file://{sdba_ds[f'qdm_{name}_hist']};"
        "group=time;"
        f"kind={quote_plus(kind)};"
        "nquantiles=50;"
        "interp=linear;")

    resp = client.get(
        f"?service=WPS&request=Execute&version=1.0.0&identifier=empirical_quantile_mapping&datainputs={datainputs}"
    )
    print(resp.response)
    assert_response_success(resp)
    out = get_output(resp.xml)
    p = xr.open_dataset(out["output"][7:])[name]

    uc = convert_calendar(u, "noleap")
    middle = ((uc > 1e-2) * (uc < 0.99)).data

    ref = xr.open_dataset(sdba_ds[f"qdm_{name}_ref"])[name]
    refc = convert_calendar(ref, "noleap")
    np.testing.assert_allclose(p[middle], refc[middle], rtol=0.03)
Example #30
0
    def test_simple(self):
        client = client_for(
            Service(processes=[
                RavenMOHYSEProcess(),
            ], cfgfiles=CFG_FILE))

        params = '1.0000, 0.0468, 4.2952, 2.6580, 0.4038, 0.0621, 0.0273, 0.0453'
        hrus = '0.9039, 5.6179775'

        datainputs = "ts=files@xlink:href=file://{ts};" \
                     "params={params};" \
                     "hrus={hrus};" \
                     "start_date={start_date};" \
                     "end_date={end_date};" \
                     "name={name};" \
                     "run_name={run_name};" \
                     "area={area};" \
                     "latitude={latitude};" \
                     "longitude={longitude};" \
                     "elevation={elevation};" \
            .format(ts=TESTDATA['raven-mohyse-nc-ts'],
                    params=params,
                    hrus=hrus,
                    start_date=dt.datetime(2000, 1, 1),
                    end_date=dt.datetime(2002, 1, 1),
                    name='Salmon',
                    run_name='test-mohyse',
                    area='4250.6',
                    elevation='843.0',
                    latitude=54.4848,
                    longitude=-123.3659,
                    )

        resp = client.get(service='WPS',
                          request='Execute',
                          version='1.0.0',
                          identifier='raven-mohyse',
                          datainputs=datainputs)

        assert_response_success(resp)
        out = get_output(resp.xml)

        assert 'diagnostics' in out
        tmp_file, _ = urlretrieve(out['diagnostics'])
        tmp_content = open(tmp_file).readlines()

        # checking correctness of NSE (full period 1954-2011 would be NSE=0.391103 as template in Wiki)
        assert 'DIAG_NASH_SUTCLIFFE' in tmp_content[0]
        idx_diag = tmp_content[0].split(',').index("DIAG_NASH_SUTCLIFFE")
        diag = np.float(tmp_content[1].split(',')[idx_diag])
        np.testing.assert_almost_equal(
            diag, 0.194612, 4, err_msg='NSE is not matching expected value')

        # checking correctness of RMSE (full period 1954-2011 would be RMSE=36.7012 as template in wiki)
        assert 'DIAG_RMSE' in tmp_content[0]
        idx_diag = tmp_content[0].split(',').index("DIAG_RMSE")
        diag = np.float(tmp_content[1].split(',')[idx_diag])
        np.testing.assert_almost_equal(
            diag, 32.2197, 4, err_msg='RMSE is not matching expected value')
Example #31
0
    def test_simple(self):
        client = client_for(
            Service(processes=[
                RavenHMETSProcess(),
            ], cfgfiles=CFG_FILE))

        params = '9.5019, 0.2774, 6.3942, 0.6884, 1.2875, 5.4134, 2.3641, 0.0973, 0.0464, 0.1998, 0.0222, -1.0919, ' \
                 '2.6851, 0.3740, 1.0000, 0.4739, 0.0114, 0.0243, 0.0069, 310.7211, 916.1947'

        datainputs = "ts=files@xlink:href=file://{ts};" \
                     "params={params};" \
                     "start_date={start_date};" \
                     "end_date={end_date};" \
                     "init={init};" \
                     "name={name};" \
                     "run_name={run_name};" \
                     "area={area};" \
                     "latitude={latitude};" \
                     "longitude={longitude};" \
                     "elevation={elevation};" \
            .format(ts=TESTDATA['raven-hmets-nc-ts'],
                    params=params,
                    start_date=dt.datetime(2000, 1, 1),
                    end_date=dt.datetime(2002, 1, 1),
                    init='155,455',
                    name='Salmon',
                    run_name='test-hmets',
                    area='4250.6',
                    elevation='843.0',
                    latitude=54.4848,
                    longitude=-123.3659,
                    )

        resp = client.get(service='WPS',
                          request='Execute',
                          version='1.0.0',
                          identifier='raven-hmets',
                          datainputs=datainputs)

        assert_response_success(resp)
        out = get_output(resp.xml)
        assert 'diagnostics' in out
        tmp_file, _ = urlretrieve(out['diagnostics'])
        tmp_content = open(tmp_file).readlines()

        # checking correctness of NSE (full period 1954-2011 would be NSE=0.636015 as template in Wiki)
        assert 'DIAG_NASH_SUTCLIFFE' in tmp_content[0]
        idx_diag = tmp_content[0].split(',').index("DIAG_NASH_SUTCLIFFE")
        diag = np.float(tmp_content[1].split(',')[idx_diag])
        np.testing.assert_almost_equal(
            diag, -7.03141, 4, err_msg='NSE is not matching expected value')

        # checking correctness of RMSE (full period 1954-2011 would be RMSE=28.3759 as template in wiki)
        assert 'DIAG_RMSE' in tmp_content[0]
        idx_diag = tmp_content[0].split(',').index("DIAG_RMSE")
        diag = np.float(tmp_content[1].split(',')[idx_diag])

        np.testing.assert_almost_equal(
            diag, 101.745, 4, err_msg='RMSE is not matching expected value')
Example #32
0
def test_wps_hello():
    client = client_for(Service(processes=[SayHello()]))
    datainputs = "name=LovelySugarBird"
    resp = client.get(
        "?service=WPS&request=Execute&version=1.0.0&identifier=hello&datainputs={}".format(
            datainputs))
    assert_response_success(resp)
    assert get_output(resp.xml) == {'output': "Hello LovelySugarBird"}
Example #33
0
def test_wps_fetch():
    client = client_for(Service(processes=[FetchProcess()]))
    datainputs = "resource=files@xlink:href={0}".format(TESTDATA['cmip5_tasmax_2006_nc'])
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='fetch_resources',
        datainputs=datainputs)
    assert_response_success(resp)
Example #34
0
def test_wps_bbox():
    client = client_for(Service(processes=[Box()]))
    datainputs = "bbox=101,42,110,46"
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='bbox',
        datainputs=datainputs)
    assert_response_success(resp)
Example #35
0
def test_wps_chomsky():
    client = client_for(Service(processes=[Chomsky()]))
    datainputs = "times=10"
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='chomsky',
        datainputs=datainputs)
    assert_response_success(resp)
Example #36
0
def test_wps_ncmeta_file():
    client = client_for(Service(processes=[NCMeta()]))
    datainputs = "dataset=@xlink:href={0}".format(NC_FILE_URL)
    resp = client.get(
        service='wps', request='execute', version='1.0.0',
        identifier='ncmeta',
        datainputs=datainputs)
    assert_response_success(resp)
def test_wps_spotchecker_file():
    client = client_for(Service(processes=[SpotChecker()]))
    datainputs = "dataset=@xlink:href={0};test=CF-1.6".format(TESTDATA['test_local_nc'])
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='spotchecker',
        datainputs=datainputs)
    assert_response_success(resp)
Example #38
0
def test_wps_thredds_download():
    client = client_for(Service(processes=[ThreddsDownload()]))
    datainputs = "url=@xlink:href={}".format(TESTDATA['noaa_catalog_1'])
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='thredds_download',
        datainputs=datainputs)
    assert_response_success(resp)
Example #39
0
def test_wps_qa_cfchecker():
    client = client_for(Service(processes=[HDHCFChecker()]))
    datainputs = "dataset@xlink:href={0};cf_version=auto".format(TESTDATA['noaa_nc_1'])
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='qa_cfchecker',
        datainputs=datainputs)
    assert_response_success(resp)
Example #40
0
def test_wps_sleep():
    client = client_for(Service(processes=[Sleep()]))
    datainputs = "delay=0.1"
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0', identifier='sleep',
        datainputs=datainputs)
    print(resp.data)
    assert_response_success(resp)
Example #41
0
def test_wps_ncdump_file():
    client = client_for(Service(processes=[NCDump()]))
    datainputs = "dataset=@xlink:href={0};".format(TESTDATA['test_local_nc'])
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='ncdump',
        datainputs=datainputs)
    assert_response_success(resp)
def test_wps_download():
    client = client_for(Service(processes=[Download()]))
    datainputs = "resource={}".format(TESTDATA['noaa_nc_1'])
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='download',
        datainputs=datainputs)
    assert_response_success(resp)
Example #43
0
 def test_post_with_no_inputs(self):
     client = client_for(Service(processes=[create_ultimate_question()]))
     request_doc = WPS.Execute(
         OWS.Identifier('ultimate_question'),
         version='1.0.0'
     )
     resp = client.post_xml(doc=request_doc)
     assert_response_success(resp)
     assert get_output(resp.xml) == {'outvalue': '42'}
Example #44
0
def test_wps_wordcount_file():
    client = client_for(Service(processes=[WordCounter()]))
    datainputs = "text=@xlink:href=file://{0}".format(
        resource_file('alice-in-wonderland.txt'))
    resp = client.get(
        service='wps', request='execute', version='1.0.0',
        identifier='wordcounter',
        datainputs=datainputs)
    assert_response_success(resp)
Example #45
0
def test_wps_poly_centroid_get():
    client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=cfgfiles))
    datainputs = "polygon=@xlink:href=file://{0}".format(resource_file('poly.xml'))
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='poly_centroid',
        datainputs=datainputs)
    assert_response_success(resp)
    assert get_output(resp.xml) == {'output': "119.59740,-13.57388"}
Example #46
0
def test_wps_dummy():
    client = client_for(Service(processes=[Dummy()]))
    datainputs = "input1=10;input2=2"
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='dummyprocess',
        datainputs=datainputs)
    assert_response_success(resp)
    assert get_output(resp.xml) == {'output1': '11', 'output2': '1'}
Example #47
0
def test_wps_inout():
    client = client_for(Service(processes=[InOut()]))
    datainputs = "string=onetwothree;int=7;float=2.0;boolean=0;text=some string;dataset=@xlink:href={}"
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0', identifier='inout',
        datainputs=datainputs.format(NC_FILE_URL))
    assert_response_success(resp)
    out = get_output(resp.xml)
    assert out['text'] == 'some string'
    assert out['dataset'].endswith('.nc')
Example #48
0
def test_wps_nonpyid():
    d = {'a': 1}
    client = client_for(Service(processes=[NonPyID()]))
    datainputs = "input 1=10;input-2={}".format(json.dumps(d))
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='non.py-id',
        datainputs=datainputs)
    assert_response_success(resp)
    assert get_output(resp.xml) == {'output.1': '11.0', 'output 2': json.dumps(d)}
def test_wps_pointinspection():
    client = client_for(
        Service(processes=[PointinspectionProcess()], cfgfiles=CFG_FILE))
    datainputs = datainputs_fmt.format(
        TESTDATA['cmip5_tasmax_2006_nc'],
        "2.356138, 48.846450",)
    resp = client.get(
        service='wps', request='execute', version='1.0.0',
        identifier='pointinspection',
        datainputs=datainputs)
    assert_response_success(resp)
def test_aggregation():
    client = client_for(Service(processes=[ESGSearchProcess()]))
    datainputs = "url={};search_type={};limit={};offset={};constraints={}".format(
        'http://esgf-data.dkrz.de/esg-search',
        'Aggregation', '5', '20',
        'project:CORDEX,time_frequency:mon,variable:tas,experiment:historical')
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='esgsearch',
        datainputs=datainputs)
    assert_response_success(resp)
Example #51
0
    def test_epsg_based_location(self):
        """Test whether the EPSG of a mapset corresponds the specified one."""
        my_process = grass_epsg_based_location()
        client = client_for(Service(processes=[my_process]))

        request_doc = WPS.Execute(
            OWS.Identifier('my_epsg_based_location'),
            version='1.0.0'
        )

        resp = client.post_xml(doc=request_doc)
        assert_response_success(resp)
def test_dataset_query():
    client = client_for(Service(processes=[ESGSearchProcess()]))
    datainputs = "url={};search_type={};limit={};offset={};constraints={};query={}".format(
        'http://esgf-data.dkrz.de/esg-search',
        'Dataset', '1', '0',
        'project:CORDEX',
        'geopotential')
    resp = client.get(
        service='WPS', request='Execute', version='1.0.0',
        identifier='esgsearch',
        datainputs=datainputs)
    assert_response_success(resp)
Example #53
0
 def test_wcs(self):
     try:
         sys.path.append("/usr/lib/grass64/etc/python/")
         import grass.script as grass
     except:
         self.skipTest("GRASS lib not found")
     client = client_for(Service(processes=[create_sum_one()]))
     request_doc = WPS.Execute(
         OWS.Identifier("sum_one"),
         WPS.DataInputs(WPS.Input(OWS.Identifier("input"), WPS.Reference(href=wcsResource, mimeType="image/tiff"))),
         WPS.ProcessOutputs(WPS.Output(OWS.Identifier("output"))),
         version="1.0.0",
     )
     resp = client.post_xml(doc=request_doc)
     assert_response_success(resp)
Example #54
0
def test_wps_poly_centroid_post():
    client = client_for(Service(processes=[PolyCentroid(), ], cfgfiles=cfgfiles))
    request_doc = WPS.Execute(
        OWS.Identifier('poly_centroid'),
        WPS.DataInputs(
            WPS.Input(
                OWS.Identifier('polygon'),
                WPS.Data(WPS.ComplexData(open(resource_file('poly.xml'), 'r').read()))
            )
        ),
        version='1.0.0'
    )
    resp = client.post_xml(doc=request_doc)
    assert_response_success(resp)
    assert get_output(resp.xml) == {'output': "119.59740,-13.57388"}
Example #55
0
 def test_post_with_string_input(self):
     client = client_for(Service(processes=[create_greeter()]))
     request_doc = WPS.Execute(
         OWS.Identifier('greeter'),
         WPS.DataInputs(
             WPS.Input(
                 OWS.Identifier('name'),
                 WPS.Data(WPS.LiteralData('foo'))
             )
         ),
         version='1.0.0'
     )
     resp = client.post_xml(doc=request_doc)
     assert_response_success(resp)
     assert get_output(resp.xml) == {'message': "Hello foo!"}
def test_wps_subset_countries():
    client = client_for(
        Service(processes=[SubsetcountryProcess()], cfgfiles=CFG_FILE))

    datainputs = datainputs_fmt.format(
        TESTDATA['cmip5_tasmax_2006_nc'],
        'CAN',
        "True")

    resp = client.get(
        service='wps', request='execute', version='1.0.0',
        identifier='subset_countries',
        datainputs=datainputs)

    assert_response_success(resp)

    # Check output file size is smaller than input.
    out = get_output(resp.xml)
    assert 'output' in out.keys()
Example #57
0
    def test_file_based_location(self):
        """Test whether the datum of a mapset corresponds the file one."""
        my_process = grass_file_based_location()
        client = client_for(Service(processes=[my_process]))

        href = 'http://demo.mapserver.org/cgi-bin/wfs?service=WFS&' \
               'version=1.1.0&request=GetFeature&typename=continents&' \
               'maxfeatures=1'

        request_doc = WPS.Execute(
            OWS.Identifier('my_file_based_location'),
            WPS.DataInputs(
                WPS.Input(
                    OWS.Identifier('input1'),
                    WPS.Reference(
                        {'{http://www.w3.org/1999/xlink}href': href}))),
            version='1.0.0')

        resp = client.post_xml(doc=request_doc)
        assert_response_success(resp)
Example #58
0
    def test_wfs(self):
        client = client_for(Service(processes=[create_feature()]))
        request_doc = WPS.Execute(
            OWS.Identifier('feature'),
            WPS.DataInputs(
                WPS.Input(
                    OWS.Identifier('input'),
                    WPS.Reference(
                        {'{http://www.w3.org/1999/xlink}href': wfsResource},
                        mimeType=FORMATS.GML.mime_type,
                        encoding='',
                        schema=''))),
            WPS.ProcessOutputs(
                WPS.Output(
                    OWS.Identifier('output'))),
            version='1.0.0'
        )
        resp = client.post_xml(doc=request_doc)

        assert_response_success(resp)
def test_wps_thredds_workflow():
    doc = """
    workflow:
      name: test_thredds_workflow
    source:
      thredds:
        catalog_url: {0}
    worker:
      identifier: dummy
      url: http://localhost:5000/wps
      resource: dataset
      inputs: []
    """.format(TESTDATA['noaa_catalog_1'])
    fp = tempfile.NamedTemporaryFile(suffix=".txt")
    yaml.dump(yaml.load(doc), fp)

    client = client_for(Service(processes=[DispelWorkflow()]))
    datainputs = "workflow@xlink:href=file://{0}".format(fp.name)
    resp = client.get(
        service='wps', request='execute', version='1.0.0', identifier='workflow',
        datainputs=datainputs)
    assert_response_success(resp)