Example #1
0
    def test_ok(self):
        job_id = self.postzip('archive_ok.zip')
        self.wait()
        log = self.get('%s/log/:' % job_id)
        self.assertGreater(len(log), 0)
        results = self.get('%s/results' % job_id)
        self.assertGreater(len(results), 0)
        for res in results:
            for etype in res['outtypes']:  # test all export types
                text = self.get_text('result/%s' % res['id'],
                                     export_type=etype)
                print('downloading result/%s' % res['id'], res['type'], etype)
                self.assertGreater(len(text), 0)

        # test no filtering in actions.get_calcs
        all_jobs = self.get('list')
        self.assertGreater(len(all_jobs), 0)

        # there is some logic in `core.export_from_db` that it is only
        # exercised when the export fails
        datadir, dskeys = actions.get_results(db, job_id)
        # try to export a non-existing output
        with self.assertRaises(core.DataStoreExportError) as ctx:
            core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp')
        self.assertIn('Could not export XXX in csv', str(ctx.exception))
    def test_ok(self):
        job_id = self.postzip('archive_ok.zip')
        self.wait()
        log = self.get('%s/log/:' % job_id)
        self.assertGreater(len(log), 0)
        results = self.get('%s/results' % job_id)
        self.assertGreater(len(results), 0)
        for res in results:
            for etype in res['outtypes']:  # test all export types
                text = self.get_text('result/%s' % res['id'],
                                     export_type=etype)
                print('downloading result/%s' % res['id'], res['type'], etype)
                self.assertGreater(len(text), 0)

        # test no filtering in actions.get_calcs
        all_jobs = self.get('list')
        self.assertGreater(len(all_jobs), 0)

        extract_url = '/v1/calc/%s/extract/' % job_id

        # check extract/composite_risk_model.attrs
        url = extract_url + 'composite_risk_model.attrs'
        self.assertEqual(self.c.get(url).status_code, 200)

        # check asset_values
        resp = self.c.get(extract_url + 'asset_values/0')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['array']), 0)  # there are 0 assets on site 0
        self.assertEqual(resp.status_code, 200)

        # check asset_tags
        resp = self.c.get(extract_url + 'asset_tags')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['taxonomy']), 7)

        # check avg_losses-rlzs
        resp = self.c.get(extract_url +
                          'agglosses/structural?taxonomy=W-SLFB-1')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['array']), 1)  # expected 1 aggregate value
        self.assertEqual(resp.status_code, 200)

        # TODO: check aggcurves

        # there is some logic in `core.export_from_db` that it is only
        # exercised when the export fails
        datadir, dskeys = actions.get_results(db, job_id)
        # try to export a non-existing output
        with self.assertRaises(core.DataStoreExportError) as ctx:
            core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp')
        self.assertIn('Could not export XXX in csv', str(ctx.exception))
Example #3
0
def get_result(request, result_id):
    """
    Download a specific result, by ``result_id``.

    The common abstracted functionality for getting hazard or risk results.

    :param request:
        `django.http.HttpRequest` object. Can contain a `export_type` GET
        param (the default is 'xml' if no param is specified).
    :param result_id:
        The id of the requested artifact.
    :returns:
        If the requested ``result_id`` is not available in the format
        designated by the `export_type`.

        Otherwise, return a `django.http.HttpResponse` containing the content
        of the requested artifact.

    Parameters for the GET request can include an `export_type`, such as 'xml',
    'geojson', 'csv', etc.
    """
    # If the result for the requested ID doesn't exist, OR
    # the job which it is related too is not complete,
    # throw back a 404.
    try:
        job_id, job_status, datadir, ds_key = logs.dbcmd(
            'get_result', result_id)
        if not job_status == 'complete':
            return HttpResponseNotFound()
    except dbapi.NotFound:
        return HttpResponseNotFound()

    etype = request.GET.get('export_type')
    export_type = etype or DEFAULT_EXPORT_TYPE

    tmpdir = tempfile.mkdtemp()
    try:
        exported = core.export_from_db(
            (ds_key, export_type), job_id, datadir, tmpdir)
    except DataStoreExportError as exc:
        # TODO: there should be a better error page
        return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
                            content_type='text/plain', status=500)
    if exported is None:
        # Throw back a 404 if the exact export parameters are not supported
        return HttpResponseNotFound(
            'export_type=%s is not supported for %s' % (export_type, ds_key))

    content_type = EXPORT_CONTENT_TYPE_MAP.get(
        export_type, DEFAULT_CONTENT_TYPE)
    try:
        fname = 'output-%s-%s' % (result_id, os.path.basename(exported))
        # 'b' is needed when running the WebUI on Windows
        data = open(exported, 'rb').read()
        response = HttpResponse(data, content_type=content_type)
        response['Content-Length'] = len(data)
        response['Content-Disposition'] = 'attachment; filename=%s' % fname
        return response
    finally:
        shutil.rmtree(tmpdir)
Example #4
0
    def test_ok(self):
        job_id = self.postzip("archive_ok.zip")
        self.wait()
        log = self.get("%s/log/:" % job_id)
        self.assertGreater(len(log), 0)
        results = self.get("%s/results" % job_id)
        self.assertGreater(len(results), 0)
        for res in results:
            for etype in res["outtypes"]:  # test all export types
                text = self.get_text("result/%s" % res["id"], export_type=etype)
                print("downloading result/%s" % res["id"], res["type"], etype)
                self.assertGreater(len(text), 0)

        # test no filtering in actions.get_calcs
        all_jobs = self.get("list")
        self.assertGreater(len(all_jobs), 0)

        # there is some logic in `core.export_from_db` that it is only
        # exercised when the export fails
        datadir, dskeys = actions.get_results(db, job_id)
        # try to export a non-existing output
        with self.assertRaises(core.DataStoreExportError) as ctx:
            core.export_from_db(("XXX", "csv"), job_id, datadir, "/tmp")
        self.assertIn("Could not export XXX in csv", str(ctx.exception))
Example #5
0
def calc_result(request, result_id):
    """
    Download a specific result, by ``result_id``.

    The common abstracted functionality for getting hazard or risk results.

    :param request:
        `django.http.HttpRequest` object. Can contain a `export_type` GET
        param (the default is 'xml' if no param is specified).
    :param result_id:
        The id of the requested artifact.
    :returns:
        If the requested ``result_id`` is not available in the format
        designated by the `export_type`.

        Otherwise, return a `django.http.HttpResponse` containing the content
        of the requested artifact.

    Parameters for the GET request can include an `export_type`, such as 'xml',
    'geojson', 'csv', etc.
    """
    # If the result for the requested ID doesn't exist, OR
    # the job which it is related too is not complete,
    # throw back a 404.
    try:
        job_id, job_status, job_user, datadir, ds_key = logs.dbcmd(
            'get_result', result_id)
        if not utils.user_has_permission(request, job_user):
            return HttpResponseForbidden()
    except dbapi.NotFound:
        return HttpResponseNotFound()

    etype = request.GET.get('export_type')
    export_type = etype or DEFAULT_EXPORT_TYPE

    tmpdir = tempfile.mkdtemp()
    try:
        exported = core.export_from_db((ds_key, export_type), job_id, datadir,
                                       tmpdir)
    except DataStoreExportError as exc:
        # TODO: there should be a better error page
        return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
                            content_type='text/plain',
                            status=500)
    if not exported:
        # Throw back a 404 if the exact export parameters are not supported
        return HttpResponseNotFound(
            'Nothing to export for export_type=%s, %s' % (export_type, ds_key))
    elif len(exported) > 1:
        # Building an archive so that there can be a single file download
        archname = ds_key + '-' + export_type + '.zip'
        zipfiles(exported, os.path.join(tmpdir, archname))
        exported = os.path.join(tmpdir, archname)
    else:  # single file
        exported = exported[0]

    content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type,
                                               DEFAULT_CONTENT_TYPE)

    fname = 'output-%s-%s' % (result_id, os.path.basename(exported))
    stream = FileWrapper(open(exported, 'rb'))  # 'b' is needed on Windows
    stream.close = lambda: (FileWrapper.close(stream), shutil.rmtree(tmpdir))
    response = FileResponse(stream, content_type=content_type)
    response['Content-Disposition'] = ('attachment; filename=%s' %
                                       os.path.basename(fname))
    response['Content-Length'] = str(os.path.getsize(exported))
    return response
Example #6
0
def get_result(request, result_id):
    """
    Download a specific result, by ``result_id``.

    The common abstracted functionality for getting hazard or risk results.

    :param request:
        `django.http.HttpRequest` object. Can contain a `export_type` GET
        param (the default is 'xml' if no param is specified).
    :param result_id:
        The id of the requested artifact.
    :returns:
        If the requested ``result_id`` is not available in the format
        designated by the `export_type`.

        Otherwise, return a `django.http.HttpResponse` containing the content
        of the requested artifact.

    Parameters for the GET request can include an `export_type`, such as 'xml',
    'geojson', 'csv', etc.
    """
    # If the result for the requested ID doesn't exist, OR
    # the job which it is related too is not complete,
    # throw back a 404.
    try:
        job_id, job_status, datadir, ds_key = logs.dbcmd(
            'get_result', result_id)
    except dbapi.NotFound:
        return HttpResponseNotFound()

    etype = request.GET.get('export_type')
    export_type = etype or DEFAULT_EXPORT_TYPE

    tmpdir = tempfile.mkdtemp()
    try:
        exported = core.export_from_db((ds_key, export_type), job_id, datadir,
                                       tmpdir)
    except DataStoreExportError as exc:
        # TODO: there should be a better error page
        return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
                            content_type='text/plain',
                            status=500)
    if exported is None:
        # Throw back a 404 if the exact export parameters are not supported
        return HttpResponseNotFound('export_type=%s is not supported for %s' %
                                    (export_type, ds_key))

    content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type,
                                               DEFAULT_CONTENT_TYPE)

    bname = os.path.basename(exported)
    if bname.startswith('.'):
        # the "." is added by `export_from_db`, strip it
        bname = bname[1:]
    fname = 'output-%s-%s' % (result_id, bname)
    # 'b' is needed when running the WebUI on Windows
    stream = FileWrapper(open(exported, 'rb'))
    stream.close = lambda: (FileWrapper.close(stream), shutil.rmtree(tmpdir))
    response = FileResponse(stream, content_type=content_type)
    response['Content-Disposition'] = ('attachment; filename=%s' %
                                       os.path.basename(fname))
    return response
Example #7
0
    def test_ok(self):
        job_id = self.postzip('archive_ok.zip')
        self.wait()
        log = self.get('%s/log/:' % job_id)
        self.assertGreater(len(log), 0)
        results = self.get('%s/results' % job_id)
        self.assertGreater(len(results), 0)
        for res in results:
            for etype in res['outtypes']:  # test all export types
                if etype == 'xml' and res['type'] == 'gmf_data':
                    continue  # do not export GMFs in XML for event based
                text = self.get_text(
                    'result/%s' % res['id'], export_type=etype)
                print('downloading result/%s' % res['id'], res['type'], etype)
                self.assertGreater(len(text), 0)

        # test no filtering in actions.get_calcs
        all_jobs = self.get('list')
        self.assertGreater(len(all_jobs), 0)

        extract_url = '/v1/calc/%s/extract/' % job_id

        # check extract/composite_risk_model.attrs
        url = extract_url + 'composite_risk_model.attrs'
        self.assertEqual(self.c.get(url).status_code, 200)

        # check asset_values
        resp = self.c.get(extract_url + 'asset_values/0')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['array']), 49)  # 49 assets on site 0
        self.assertEqual(resp.status_code, 200)

        # check asset_tags
        resp = self.c.get(extract_url + 'asset_tags')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['taxonomy']), 7)

        # check avg_losses-rlzs
        resp = self.c.get(
            extract_url + 'agg_losses/structural?taxonomy=W-SLFB-1')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['array']), 1)  # expected 1 aggregate value
        self.assertEqual(resp.status_code, 200)

        # check *-aggregation
        resp = self.c.get(
            extract_url + 'agg_losses/structural?taxonomy=*')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['tags']), 6)  # expected 6 taxonomies
        self.assertEqual(len(got['array']), 6)  # expected 6 aggregates
        self.assertEqual(resp.status_code, 200)

        # check agg_curves with a single realization
        # the case with multiple rlzs is tested in event_based_risk/case_master
        resp = self.c.get(
            extract_url + 'agg_curves/structural?taxonomy=*')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(list(got['stats']), [b'mean'])
        self.assertEqual(list(got['return_periods']), [5, 10, 20, 50, 100])
        self.assertEqual(list(got['units']), [b'EUR'])

        # there is some logic in `core.export_from_db` that it is only
        # exercised when the export fails
        datadir, dskeys = actions.get_results(db, job_id)
        # try to export a non-existing output
        with self.assertRaises(core.DataStoreExportError) as ctx:
            core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp')
        self.assertIn('Could not export XXX in csv', str(ctx.exception))

        # check MFD distribution
        extract_url = '/v1/calc/%s/extract/event_based_mfd' % job_id
        data = b''.join(ln for ln in self.c.get(extract_url))
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertGreater(len(got['array']['mag']), 1)
        self.assertGreater(len(got['array']['freq']), 1)
Example #8
0
def calc_result(request, result_id):
    """
    Download a specific result, by ``result_id``.

    The common abstracted functionality for getting hazard or risk results.

    :param request:
        `django.http.HttpRequest` object. Can contain a `export_type` GET
        param (the default is 'xml' if no param is specified).
    :param result_id:
        The id of the requested artifact.
    :returns:
        If the requested ``result_id`` is not available in the format
        designated by the `export_type`.

        Otherwise, return a `django.http.HttpResponse` containing the content
        of the requested artifact.

    Parameters for the GET request can include an `export_type`, such as 'xml',
    'geojson', 'csv', etc.
    """
    # If the result for the requested ID doesn't exist, OR
    # the job which it is related too is not complete,
    # throw back a 404.
    try:
        job_id, job_status, job_user, datadir, ds_key = logs.dbcmd(
            'get_result', result_id)
        if not utils.user_has_permission(request, job_user):
            return HttpResponseForbidden()
    except dbapi.NotFound:
        return HttpResponseNotFound()

    etype = request.GET.get('export_type')
    export_type = etype or DEFAULT_EXPORT_TYPE

    tmpdir = tempfile.mkdtemp()
    try:
        exported = core.export_from_db(
            (ds_key, export_type), job_id, datadir, tmpdir)
    except DataStoreExportError as exc:
        # TODO: there should be a better error page
        return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc),
                            content_type='text/plain', status=500)
    if not exported:
        # Throw back a 404 if the exact export parameters are not supported
        return HttpResponseNotFound(
            'Nothing to export for export_type=%s, %s' % (export_type, ds_key))
    elif len(exported) > 1:
        # Building an archive so that there can be a single file download
        archname = ds_key + '-' + export_type + '.zip'
        zipfiles(exported, os.path.join(tmpdir, archname))
        exported = os.path.join(tmpdir, archname)
    else:  # single file
        exported = exported[0]

    content_type = EXPORT_CONTENT_TYPE_MAP.get(
        export_type, DEFAULT_CONTENT_TYPE)

    fname = 'output-%s-%s' % (result_id, os.path.basename(exported))
    stream = FileWrapper(open(exported, 'rb'))  # 'b' is needed on Windows
    stream.close = lambda: (
        FileWrapper.close(stream), shutil.rmtree(tmpdir))
    response = FileResponse(stream, content_type=content_type)
    response['Content-Disposition'] = (
        'attachment; filename=%s' % os.path.basename(fname))
    response['Content-Length'] = str(os.path.getsize(exported))
    return response
Example #9
0
    def test_ok(self):
        job_id = self.postzip('archive_ok.zip')
        self.wait()
        log = self.get('%s/log/:' % job_id)
        self.assertGreater(len(log), 0)
        results = self.get('%s/results' % job_id)
        self.assertGreater(len(results), 0)
        for res in results:
            for etype in res['outtypes']:  # test all export types
                if etype == 'xml' and res['type'] == 'gmf_data':
                    continue  # do not export GMFs in XML for event based
                text = self.get_text('result/%s' % res['id'],
                                     export_type=etype)
                print('downloading result/%s' % res['id'], res['type'], etype)
                self.assertGreater(len(text), 0)

        # test no filtering in actions.get_calcs
        all_jobs = self.get('list')
        self.assertGreater(len(all_jobs), 0)

        extract_url = '/v1/calc/%s/extract/' % job_id

        # check extract/composite_risk_model.attrs
        url = extract_url + 'composite_risk_model.attrs'
        self.assertEqual(self.c.get(url).status_code, 200)

        # check asset_tags
        resp = self.c.get(extract_url + 'asset_tags')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['taxonomy']), 7)

        # check exposure_metadata
        resp = self.c.get(extract_url + 'exposure_metadata')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(sorted(got['tagnames']), ['id', 'taxonomy'])
        self.assertEqual(sorted(got['array']), ['number', 'value-structural'])

        # check assets
        resp = self.c.get(extract_url +
                          'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['array']), 25)

        # check avg_losses-rlzs
        resp = self.c.get(extract_url +
                          'agg_losses/structural?taxonomy=W-SLFB-1')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['array']), 1)  # expected 1 aggregate value
        self.assertEqual(resp.status_code, 200)

        # check *-aggregation
        resp = self.c.get(extract_url + 'agg_losses/structural?taxonomy=*')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['tags']), 6)  # expected 6 taxonomies
        self.assertEqual(len(got['array']), 6)  # expected 6 aggregates
        self.assertEqual(resp.status_code, 200)

        # there is some logic in `core.export_from_db` that it is only
        # exercised when the export fails
        datadir, dskeys = actions.get_results(db, job_id)
        # try to export a non-existing output
        with self.assertRaises(core.DataStoreExportError) as ctx:
            core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp')
        self.assertIn('Could not export XXX in csv', str(ctx.exception))

        # check MFD distribution
        extract_url = '/v1/calc/%s/extract/event_based_mfd' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertGreater(len(got['array']['mag']), 1)
        self.assertGreater(len(got['array']['freq']), 1)
Example #10
0
    def test_ok(self):
        job_id = self.postzip('archive_ok.zip')
        self.wait()
        log = self.get('%s/log/:' % job_id)
        self.assertGreater(len(log), 0)
        results = self.get('%s/results' % job_id)
        self.assertGreater(len(results), 0)
        for res in results:
            for etype in res['outtypes']:  # test all export types
                text = self.get_text('result/%s' % res['id'],
                                     export_type=etype)
                print('downloading result/%s' % res['id'], res['type'], etype)
                self.assertGreater(len(text), 0)

        # test no filtering in actions.get_calcs
        all_jobs = self.get('list')
        self.assertGreater(len(all_jobs), 0)

        extract_url = '/v1/calc/%s/extract/' % job_id

        # check extract/composite_risk_model.attrs
        url = extract_url + 'composite_risk_model.attrs'
        self.assertEqual(self.c.get(url).status_code, 200)

        # check asset_tags
        resp = self.c.get(extract_url + 'asset_tags')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['taxonomy']), 7)

        # check exposure_metadata
        resp = self.c.get(extract_url + 'exposure_metadata')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(sorted(got['tagnames']), [b'taxonomy'])
        self.assertEqual(sorted(got['array']), ['number', 'value-structural'])

        # check assets
        resp = self.c.get(extract_url +
                          'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['array']), 25)

        # check avg_losses-rlzs
        resp = self.c.get(extract_url +
                          'agg_losses/structural?taxonomy=W-SLFB-1')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['array']), 1)  # expected 1 aggregate value
        self.assertEqual(resp.status_code, 200)

        # check *-aggregation
        resp = self.c.get(extract_url + 'agg_losses/structural?taxonomy=*')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['tags']), 6)  # expected 6 taxonomies
        self.assertEqual(len(got['array']), 6)  # expected 6 aggregates
        self.assertEqual(resp.status_code, 200)

        # there is some logic in `core.export_from_db` that it is only
        # exercised when the export fails
        datadir, dskeys = actions.get_results(db, job_id)
        # try to export a non-existing output
        with self.assertRaises(core.DataStoreExportError) as ctx:
            core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp')
        self.assertIn('Could not export XXX in csv', str(ctx.exception))

        # check MFD distribution
        extract_url = '/v1/calc/%s/extract/event_based_mfd?kind=mean' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertGreater(len(got['magnitudes']), 1)
        self.assertGreater(len(got['mean_frequency']), 1)

        # check rupture_info
        extract_url = '/v1/calc/%s/extract/rupture_info' % job_id
        got = loadnpz(self.c.get(extract_url))
        boundaries = gzip.decompress(got['boundaries']).split(b'\n')
        self.assertEqual(len(boundaries), 33)
        self.assertEqual(
            boundaries[0],
            b'POLYGON((-77.10575 18.83643, -77.11150 18.75286, -77.18793 18.75618, -77.18146 18.84064, -77.10575 18.83643))'
        )
        self.assertEqual(
            boundaries[32],
            b'POLYGON((-77.36446 18.50400, -77.37234 18.41776, -77.38020 18.33151, -77.38806 18.24526, -77.39591 18.15902, -77.40376 18.07277, -77.41159 17.98652, -77.41942 17.90027, -77.42724 17.81402, -77.43505 17.72777, -77.52006 17.72029, -77.60506 17.71277, -77.69004 17.70522, -77.77502 17.69763, -77.76438 17.78745, -77.75372 17.87728, -77.74306 17.96710, -77.73238 18.05692, -77.72169 18.14674, -77.71099 18.23656, -77.70028 18.32638, -77.68955 18.41620, -77.67883 18.50602, -77.60023 18.50556, -77.52164 18.50508, -77.44305 18.50455, -77.36446 18.50400))'
        )

        # check num_events
        extract_url = '/v1/calc/%s/extract/num_events' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(got['num_events'], 34)

        # check gmf_data
        extract_url = '/v1/calc/%s/extract/gmf_data?event_id=28' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(len(got['rlz-000']), 3)

        # check gmf_data with no data
        extract_url = '/v1/calc/%s/extract/gmf_data?event_id=0' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(len(got['rlz-000']), 0)

        # check extract_sources
        extract_url = '/v1/calc/%s/extract/sources?' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(list(got), ['wkt_gz', 'src_gz', 'array'])
        self.assertGreater(len(got['array']), 0)
Example #11
0
    def test_ok(self):
        job_id = self.postzip('archive_ok.zip')
        self.wait()
        log = self.get('%s/log/:' % job_id)
        self.assertGreater(len(log), 0)
        results = self.get('%s/results' % job_id)
        self.assertGreater(len(results), 0)
        for res in results:
            for etype in res['outtypes']:  # test all export types
                text = self.get_text(
                    'result/%s' % res['id'], export_type=etype)
                print('downloading result/%s' % res['id'], res['type'], etype)
                self.assertGreater(len(text), 0)

        # test no filtering in actions.get_calcs
        all_jobs = self.get('list')
        self.assertGreater(len(all_jobs), 0)

        extract_url = '/v1/calc/%s/extract/' % job_id

        # check extract/composite_risk_model.attrs
        url = extract_url + 'composite_risk_model.attrs'
        self.assertEqual(self.c.get(url).status_code, 200)

        # check asset_tags
        resp = self.c.get(extract_url + 'asset_tags')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['taxonomy']), 7)

        # check exposure_metadata
        resp = self.c.get(extract_url + 'exposure_metadata')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(sorted(got['tagnames']), [b'id', b'taxonomy'])
        self.assertEqual(sorted(got['array']), ['number', 'value-structural'])

        # check assets
        resp = self.c.get(
            extract_url + 'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['array']), 25)

        # check avg_losses-rlzs
        resp = self.c.get(
            extract_url + 'agg_losses/structural?taxonomy=W-SLFB-1')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['array']), 1)  # expected 1 aggregate value
        self.assertEqual(resp.status_code, 200)

        # check *-aggregation
        resp = self.c.get(
            extract_url + 'agg_losses/structural?taxonomy=*')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['tags']), 6)  # expected 6 taxonomies
        self.assertEqual(len(got['array']), 6)  # expected 6 aggregates
        self.assertEqual(resp.status_code, 200)

        # there is some logic in `core.export_from_db` that it is only
        # exercised when the export fails
        datadir, dskeys = actions.get_results(db, job_id)
        # try to export a non-existing output
        with self.assertRaises(core.DataStoreExportError) as ctx:
            core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp')
        self.assertIn('Could not export XXX in csv', str(ctx.exception))

        # check MFD distribution
        extract_url = '/v1/calc/%s/extract/event_based_mfd?kind=mean' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertGreater(len(got['magnitudes']), 1)
        self.assertGreater(len(got['mean_frequency']), 1)

        # check rupture_info
        extract_url = '/v1/calc/%s/extract/rupture_info' % job_id
        got = loadnpz(self.c.get(extract_url))
        boundaries = gzip.decompress(got['boundaries']).split(b'\n')
        self.assertEqual(len(boundaries), 33)

        # check num_events
        extract_url = '/v1/calc/%s/extract/num_events' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(got['num_events'], 34)

        # check gmf_data
        extract_url = '/v1/calc/%s/extract/gmf_data?event_id=28' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(len(got['rlz-000']), 3)

        # check gmf_data with no data
        extract_url = '/v1/calc/%s/extract/gmf_data?event_id=0' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(len(got['rlz-000']), 0)

        # check extract_sources
        extract_url = '/v1/calc/%s/extract/sources?sm_id=0' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(list(got), ['sm_id', 'wkt_gz', 'src_gz', 'array'])
        self.assertGreater(len(got['array']), 0)
Example #12
0
    def test_ok(self):
        job_id = self.postzip('archive_ok.zip')
        self.wait()
        log = self.get('%s/log/:' % job_id)
        self.assertGreater(len(log), 0)
        results = self.get('%s/results' % job_id)
        self.assertGreater(len(results), 0)
        for res in results:
            for etype in res['outtypes']:  # test all export types
                if etype == 'xml' and res['type'] == 'gmf_data':
                    continue  # do not export GMFs in XML for event based
                text = self.get_text(
                    'result/%s' % res['id'], export_type=etype)
                print('downloading result/%s' % res['id'], res['type'], etype)
                self.assertGreater(len(text), 0)

        # test no filtering in actions.get_calcs
        all_jobs = self.get('list')
        self.assertGreater(len(all_jobs), 0)

        extract_url = '/v1/calc/%s/extract/' % job_id

        # check extract/composite_risk_model.attrs
        url = extract_url + 'composite_risk_model.attrs'
        self.assertEqual(self.c.get(url).status_code, 200)

        # check asset_values
        resp = self.c.get(extract_url + 'asset_values/0')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['array']), 49)  # 49 assets on site 0
        self.assertEqual(resp.status_code, 200)

        # check asset_tags
        resp = self.c.get(extract_url + 'asset_tags')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['taxonomy']), 7)

        # check exposure_metadata
        resp = self.c.get(extract_url + 'exposure_metadata')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(list(got['tagnames']), ['taxonomy'])
        self.assertEqual(list(got['array']), ['number', 'value-structural'])

        # check assets
        resp = self.c.get(
            extract_url + 'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['array']), 25)

        # check avg_losses-rlzs
        resp = self.c.get(
            extract_url + 'agg_losses/structural?taxonomy=W-SLFB-1')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['array']), 1)  # expected 1 aggregate value
        self.assertEqual(resp.status_code, 200)

        # check *-aggregation
        resp = self.c.get(
            extract_url + 'agg_losses/structural?taxonomy=*')
        data = b''.join(ln for ln in resp.streaming_content)
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertEqual(len(got['tags']), 6)  # expected 6 taxonomies
        self.assertEqual(len(got['array']), 6)  # expected 6 aggregates
        self.assertEqual(resp.status_code, 200)

        # there is some logic in `core.export_from_db` that it is only
        # exercised when the export fails
        datadir, dskeys = actions.get_results(db, job_id)
        # try to export a non-existing output
        with self.assertRaises(core.DataStoreExportError) as ctx:
            core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp')
        self.assertIn('Could not export XXX in csv', str(ctx.exception))

        # check MFD distribution
        extract_url = '/v1/calc/%s/extract/event_based_mfd' % job_id
        data = b''.join(ln for ln in self.c.get(extract_url))
        got = numpy.load(io.BytesIO(data))  # load npz file
        self.assertGreater(len(got['array']['mag']), 1)
        self.assertGreater(len(got['array']['freq']), 1)
Example #13
0
    def test_ok(self):
        job_id = self.postzip('archive_ok.zip')
        self.wait()
        log = self.get('%s/log/:' % job_id)
        self.assertGreater(len(log), 0)
        results = self.get('%s/results' % job_id)
        self.assertGreater(len(results), 0)
        for res in results:
            for etype in res['outtypes']:  # test all export types
                text = self.get_text('result/%s' % res['id'],
                                     export_type=etype)
                print('downloading result/%s' % res['id'], res['type'], etype)
                self.assertGreater(len(text), 0)

        # test no filtering in actions.get_calcs
        all_jobs = self.get('list')
        self.assertGreater(len(all_jobs), 0)

        extract_url = '/v1/calc/%s/extract/' % job_id

        # check eids_by_gsim
        resp = self.c.get(extract_url + 'eids_by_gsim')
        dic = dict(loadnpz(resp.streaming_content))
        self.assertEqual(len(dic['[AtkinsonBoore2003SInter]']), 7)

        # check extract/composite_risk_model.attrs
        url = extract_url + 'composite_risk_model.attrs'
        self.assertEqual(self.c.get(url).status_code, 200)

        # check asset_tags
        resp = self.c.get(extract_url + 'asset_tags')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['taxonomy']), 7)

        # check exposure_metadata
        resp = self.c.get(extract_url + 'exposure_metadata')
        got = loadnpz(resp.streaming_content)['json']
        dic = json.loads(bytes(got))
        self.assertEqual(sorted(dic['tagnames']), ['taxonomy'])
        self.assertEqual(sorted(dic['names']), ['number', 'value-structural'])

        # check assets
        resp = self.c.get(extract_url +
                          'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1')
        if resp.status_code == 500:  # should never happen
            raise RuntimeError(resp.content.decode('utf8'))
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['array']), 25)

        # check losses_by_asset
        resp = self.c.get(extract_url + 'losses_by_asset')
        if resp.status_code == 500:  # should never happen
            raise RuntimeError(resp.content.decode('utf8'))
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['rlz-000']), 95)

        # check agg_losses
        resp = self.c.get(extract_url +
                          'agg_losses/structural?taxonomy=W-SLFB-1')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['array']), 1)  # expected 1 aggregate value
        self.assertEqual(resp.status_code, 200)

        # check *-aggregation
        resp = self.c.get(extract_url + 'agg_losses/structural?taxonomy=*')
        got = loadnpz(resp.streaming_content)
        self.assertEqual(len(got['tags']), 6)  # expected 6 taxonomies
        self.assertEqual(len(got['array']), 6)  # expected 6 aggregates
        self.assertEqual(resp.status_code, 200)

        # there is some logic in `core.export_from_db` that it is only
        # exercised when the export fails
        datadir, dskeys = actions.get_results(db, job_id)
        # try to export a non-existing output
        with self.assertRaises(core.DataStoreExportError) as ctx:
            core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp')
        self.assertIn('Could not export XXX in csv', str(ctx.exception))

        # check MFD distribution
        extract_url = '/v1/calc/%s/extract/event_based_mfd?kind=mean' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertGreater(len(got['magnitudes']), 1)
        self.assertGreater(len(got['mean_frequency']), 1)

        # check rupture_info
        extract_url = '/v1/calc/%s/extract/rupture_info' % job_id
        got = loadnpz(self.c.get(extract_url))
        boundaries = gzip.decompress(got['boundaries']).split(b'\n')
        self.assertEqual(len(boundaries), 37)
        self.assertEqual(
            boundaries[0],
            b'POLYGON((-77.24583 17.99602, -77.25224 17.91156, -77.33583 17.90593, -77.32871 17.99129, -77.24583 17.99602))'
        )
        self.assertEqual(
            boundaries[-1],
            b'POLYGON((-77.10000 18.92000, -77.10575 18.83643, -77.11150 18.75286, -77.11723 18.66929, -77.12297 18.58572, -77.12869 18.50215, -77.13442 18.41858, -77.14014 18.33500, -77.14584 18.25143, -77.15155 18.16786, -77.15725 18.08429, -77.16295 18.00072, -77.16864 17.91714, -77.17432 17.83357, -77.18000 17.75000, -77.26502 17.74263, -77.35004 17.73522, -77.43505 17.72777, -77.52006 17.72029, -77.60506 17.71277, -77.69004 17.70522, -77.77502 17.69763, -77.86000 17.69000, -77.84865 17.78072, -77.83729 17.87144, -77.82591 17.96215, -77.81452 18.05287, -77.80312 18.14359, -77.79172 18.23430, -77.78030 18.32502, -77.76886 18.41573, -77.75742 18.50644, -77.74596 18.59716, -77.73448 18.68787, -77.72300 18.77858, -77.71151 18.86929, -77.70000 18.96000, -77.62498 18.95510, -77.54997 18.95018, -77.47497 18.94523, -77.39996 18.94024, -77.32497 18.93523, -77.24997 18.93018, -77.17499 18.92511, -77.10000 18.92000))'
        )

        # check num_events
        extract_url = '/v1/calc/%s/extract/num_events' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(got['num_events'], 41)

        # check gmf_data with no data
        extract_url = '/v1/calc/%s/extract/gmf_data?event_id=28' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(len(got['rlz-000']), 0)

        # check extract_sources
        extract_url = '/v1/calc/%s/extract/sources?' % job_id
        got = loadnpz(self.c.get(extract_url))
        self.assertEqual(list(got), ['wkt_gz', 'src_gz', 'array'])
        self.assertGreater(len(got['array']), 0)