def test_ok(self): job_id = self.postzip('archive_ok.zip') self.wait() log = self.get('%s/log/:' % job_id) self.assertGreater(len(log), 0) results = self.get('%s/results' % job_id) self.assertGreater(len(results), 0) for res in results: for etype in res['outtypes']: # test all export types text = self.get_text('result/%s' % res['id'], export_type=etype) print('downloading result/%s' % res['id'], res['type'], etype) self.assertGreater(len(text), 0) # test no filtering in actions.get_calcs all_jobs = self.get('list') self.assertGreater(len(all_jobs), 0) # there is some logic in `core.export_from_db` that it is only # exercised when the export fails datadir, dskeys = actions.get_results(db, job_id) # try to export a non-existing output with self.assertRaises(core.DataStoreExportError) as ctx: core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp') self.assertIn('Could not export XXX in csv', str(ctx.exception))
def test_ok(self): job_id = self.postzip('archive_ok.zip') self.wait() log = self.get('%s/log/:' % job_id) self.assertGreater(len(log), 0) results = self.get('%s/results' % job_id) self.assertGreater(len(results), 0) for res in results: for etype in res['outtypes']: # test all export types text = self.get_text('result/%s' % res['id'], export_type=etype) print('downloading result/%s' % res['id'], res['type'], etype) self.assertGreater(len(text), 0) # test no filtering in actions.get_calcs all_jobs = self.get('list') self.assertGreater(len(all_jobs), 0) extract_url = '/v1/calc/%s/extract/' % job_id # check extract/composite_risk_model.attrs url = extract_url + 'composite_risk_model.attrs' self.assertEqual(self.c.get(url).status_code, 200) # check asset_values resp = self.c.get(extract_url + 'asset_values/0') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['array']), 0) # there are 0 assets on site 0 self.assertEqual(resp.status_code, 200) # check asset_tags resp = self.c.get(extract_url + 'asset_tags') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['taxonomy']), 7) # check avg_losses-rlzs resp = self.c.get(extract_url + 'agglosses/structural?taxonomy=W-SLFB-1') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['array']), 1) # expected 1 aggregate value self.assertEqual(resp.status_code, 200) # TODO: check aggcurves # there is some logic in `core.export_from_db` that it is only # exercised when the export fails datadir, dskeys = actions.get_results(db, job_id) # try to export a non-existing output with self.assertRaises(core.DataStoreExportError) as ctx: core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp') self.assertIn('Could not export XXX in csv', str(ctx.exception))
def get_result(request, result_id): """ Download a specific result, by ``result_id``. The common abstracted functionality for getting hazard or risk results. :param request: `django.http.HttpRequest` object. Can contain a `export_type` GET param (the default is 'xml' if no param is specified). :param result_id: The id of the requested artifact. :returns: If the requested ``result_id`` is not available in the format designated by the `export_type`. Otherwise, return a `django.http.HttpResponse` containing the content of the requested artifact. Parameters for the GET request can include an `export_type`, such as 'xml', 'geojson', 'csv', etc. """ # If the result for the requested ID doesn't exist, OR # the job which it is related too is not complete, # throw back a 404. try: job_id, job_status, datadir, ds_key = logs.dbcmd( 'get_result', result_id) if not job_status == 'complete': return HttpResponseNotFound() except dbapi.NotFound: return HttpResponseNotFound() etype = request.GET.get('export_type') export_type = etype or DEFAULT_EXPORT_TYPE tmpdir = tempfile.mkdtemp() try: exported = core.export_from_db( (ds_key, export_type), job_id, datadir, tmpdir) except DataStoreExportError as exc: # TODO: there should be a better error page return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc), content_type='text/plain', status=500) if exported is None: # Throw back a 404 if the exact export parameters are not supported return HttpResponseNotFound( 'export_type=%s is not supported for %s' % (export_type, ds_key)) content_type = EXPORT_CONTENT_TYPE_MAP.get( export_type, DEFAULT_CONTENT_TYPE) try: fname = 'output-%s-%s' % (result_id, os.path.basename(exported)) # 'b' is needed when running the WebUI on Windows data = open(exported, 'rb').read() response = HttpResponse(data, content_type=content_type) response['Content-Length'] = len(data) response['Content-Disposition'] = 'attachment; filename=%s' % fname return response finally: shutil.rmtree(tmpdir)
def test_ok(self): job_id = self.postzip("archive_ok.zip") self.wait() log = self.get("%s/log/:" % job_id) self.assertGreater(len(log), 0) results = self.get("%s/results" % job_id) self.assertGreater(len(results), 0) for res in results: for etype in res["outtypes"]: # test all export types text = self.get_text("result/%s" % res["id"], export_type=etype) print("downloading result/%s" % res["id"], res["type"], etype) self.assertGreater(len(text), 0) # test no filtering in actions.get_calcs all_jobs = self.get("list") self.assertGreater(len(all_jobs), 0) # there is some logic in `core.export_from_db` that it is only # exercised when the export fails datadir, dskeys = actions.get_results(db, job_id) # try to export a non-existing output with self.assertRaises(core.DataStoreExportError) as ctx: core.export_from_db(("XXX", "csv"), job_id, datadir, "/tmp") self.assertIn("Could not export XXX in csv", str(ctx.exception))
def calc_result(request, result_id): """ Download a specific result, by ``result_id``. The common abstracted functionality for getting hazard or risk results. :param request: `django.http.HttpRequest` object. Can contain a `export_type` GET param (the default is 'xml' if no param is specified). :param result_id: The id of the requested artifact. :returns: If the requested ``result_id`` is not available in the format designated by the `export_type`. Otherwise, return a `django.http.HttpResponse` containing the content of the requested artifact. Parameters for the GET request can include an `export_type`, such as 'xml', 'geojson', 'csv', etc. """ # If the result for the requested ID doesn't exist, OR # the job which it is related too is not complete, # throw back a 404. try: job_id, job_status, job_user, datadir, ds_key = logs.dbcmd( 'get_result', result_id) if not utils.user_has_permission(request, job_user): return HttpResponseForbidden() except dbapi.NotFound: return HttpResponseNotFound() etype = request.GET.get('export_type') export_type = etype or DEFAULT_EXPORT_TYPE tmpdir = tempfile.mkdtemp() try: exported = core.export_from_db((ds_key, export_type), job_id, datadir, tmpdir) except DataStoreExportError as exc: # TODO: there should be a better error page return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc), content_type='text/plain', status=500) if not exported: # Throw back a 404 if the exact export parameters are not supported return HttpResponseNotFound( 'Nothing to export for export_type=%s, %s' % (export_type, ds_key)) elif len(exported) > 1: # Building an archive so that there can be a single file download archname = ds_key + '-' + export_type + '.zip' zipfiles(exported, os.path.join(tmpdir, archname)) exported = os.path.join(tmpdir, archname) else: # single file exported = exported[0] content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type, DEFAULT_CONTENT_TYPE) fname = 'output-%s-%s' % (result_id, os.path.basename(exported)) stream = FileWrapper(open(exported, 'rb')) # 'b' is needed on Windows stream.close = lambda: (FileWrapper.close(stream), shutil.rmtree(tmpdir)) response = FileResponse(stream, content_type=content_type) response['Content-Disposition'] = ('attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(exported)) return response
def get_result(request, result_id): """ Download a specific result, by ``result_id``. The common abstracted functionality for getting hazard or risk results. :param request: `django.http.HttpRequest` object. Can contain a `export_type` GET param (the default is 'xml' if no param is specified). :param result_id: The id of the requested artifact. :returns: If the requested ``result_id`` is not available in the format designated by the `export_type`. Otherwise, return a `django.http.HttpResponse` containing the content of the requested artifact. Parameters for the GET request can include an `export_type`, such as 'xml', 'geojson', 'csv', etc. """ # If the result for the requested ID doesn't exist, OR # the job which it is related too is not complete, # throw back a 404. try: job_id, job_status, datadir, ds_key = logs.dbcmd( 'get_result', result_id) except dbapi.NotFound: return HttpResponseNotFound() etype = request.GET.get('export_type') export_type = etype or DEFAULT_EXPORT_TYPE tmpdir = tempfile.mkdtemp() try: exported = core.export_from_db((ds_key, export_type), job_id, datadir, tmpdir) except DataStoreExportError as exc: # TODO: there should be a better error page return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc), content_type='text/plain', status=500) if exported is None: # Throw back a 404 if the exact export parameters are not supported return HttpResponseNotFound('export_type=%s is not supported for %s' % (export_type, ds_key)) content_type = EXPORT_CONTENT_TYPE_MAP.get(export_type, DEFAULT_CONTENT_TYPE) bname = os.path.basename(exported) if bname.startswith('.'): # the "." is added by `export_from_db`, strip it bname = bname[1:] fname = 'output-%s-%s' % (result_id, bname) # 'b' is needed when running the WebUI on Windows stream = FileWrapper(open(exported, 'rb')) stream.close = lambda: (FileWrapper.close(stream), shutil.rmtree(tmpdir)) response = FileResponse(stream, content_type=content_type) response['Content-Disposition'] = ('attachment; filename=%s' % os.path.basename(fname)) return response
def test_ok(self): job_id = self.postzip('archive_ok.zip') self.wait() log = self.get('%s/log/:' % job_id) self.assertGreater(len(log), 0) results = self.get('%s/results' % job_id) self.assertGreater(len(results), 0) for res in results: for etype in res['outtypes']: # test all export types if etype == 'xml' and res['type'] == 'gmf_data': continue # do not export GMFs in XML for event based text = self.get_text( 'result/%s' % res['id'], export_type=etype) print('downloading result/%s' % res['id'], res['type'], etype) self.assertGreater(len(text), 0) # test no filtering in actions.get_calcs all_jobs = self.get('list') self.assertGreater(len(all_jobs), 0) extract_url = '/v1/calc/%s/extract/' % job_id # check extract/composite_risk_model.attrs url = extract_url + 'composite_risk_model.attrs' self.assertEqual(self.c.get(url).status_code, 200) # check asset_values resp = self.c.get(extract_url + 'asset_values/0') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['array']), 49) # 49 assets on site 0 self.assertEqual(resp.status_code, 200) # check asset_tags resp = self.c.get(extract_url + 'asset_tags') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['taxonomy']), 7) # check avg_losses-rlzs resp = self.c.get( extract_url + 'agg_losses/structural?taxonomy=W-SLFB-1') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['array']), 1) # expected 1 aggregate value self.assertEqual(resp.status_code, 200) # check *-aggregation resp = self.c.get( extract_url + 'agg_losses/structural?taxonomy=*') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['tags']), 6) # expected 6 taxonomies self.assertEqual(len(got['array']), 6) # expected 6 aggregates self.assertEqual(resp.status_code, 200) # check agg_curves with a single realization # the case with multiple rlzs is tested in event_based_risk/case_master resp = self.c.get( extract_url + 'agg_curves/structural?taxonomy=*') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(list(got['stats']), [b'mean']) self.assertEqual(list(got['return_periods']), [5, 10, 20, 50, 100]) self.assertEqual(list(got['units']), [b'EUR']) # there is some logic in `core.export_from_db` that it is only # exercised when the export fails datadir, dskeys = actions.get_results(db, job_id) # try to export a non-existing output with self.assertRaises(core.DataStoreExportError) as ctx: core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp') self.assertIn('Could not export XXX in csv', str(ctx.exception)) # check MFD distribution extract_url = '/v1/calc/%s/extract/event_based_mfd' % job_id data = b''.join(ln for ln in self.c.get(extract_url)) got = numpy.load(io.BytesIO(data)) # load npz file self.assertGreater(len(got['array']['mag']), 1) self.assertGreater(len(got['array']['freq']), 1)
def calc_result(request, result_id): """ Download a specific result, by ``result_id``. The common abstracted functionality for getting hazard or risk results. :param request: `django.http.HttpRequest` object. Can contain a `export_type` GET param (the default is 'xml' if no param is specified). :param result_id: The id of the requested artifact. :returns: If the requested ``result_id`` is not available in the format designated by the `export_type`. Otherwise, return a `django.http.HttpResponse` containing the content of the requested artifact. Parameters for the GET request can include an `export_type`, such as 'xml', 'geojson', 'csv', etc. """ # If the result for the requested ID doesn't exist, OR # the job which it is related too is not complete, # throw back a 404. try: job_id, job_status, job_user, datadir, ds_key = logs.dbcmd( 'get_result', result_id) if not utils.user_has_permission(request, job_user): return HttpResponseForbidden() except dbapi.NotFound: return HttpResponseNotFound() etype = request.GET.get('export_type') export_type = etype or DEFAULT_EXPORT_TYPE tmpdir = tempfile.mkdtemp() try: exported = core.export_from_db( (ds_key, export_type), job_id, datadir, tmpdir) except DataStoreExportError as exc: # TODO: there should be a better error page return HttpResponse(content='%s: %s' % (exc.__class__.__name__, exc), content_type='text/plain', status=500) if not exported: # Throw back a 404 if the exact export parameters are not supported return HttpResponseNotFound( 'Nothing to export for export_type=%s, %s' % (export_type, ds_key)) elif len(exported) > 1: # Building an archive so that there can be a single file download archname = ds_key + '-' + export_type + '.zip' zipfiles(exported, os.path.join(tmpdir, archname)) exported = os.path.join(tmpdir, archname) else: # single file exported = exported[0] content_type = EXPORT_CONTENT_TYPE_MAP.get( export_type, DEFAULT_CONTENT_TYPE) fname = 'output-%s-%s' % (result_id, os.path.basename(exported)) stream = FileWrapper(open(exported, 'rb')) # 'b' is needed on Windows stream.close = lambda: ( FileWrapper.close(stream), shutil.rmtree(tmpdir)) response = FileResponse(stream, content_type=content_type) response['Content-Disposition'] = ( 'attachment; filename=%s' % os.path.basename(fname)) response['Content-Length'] = str(os.path.getsize(exported)) return response
def test_ok(self): job_id = self.postzip('archive_ok.zip') self.wait() log = self.get('%s/log/:' % job_id) self.assertGreater(len(log), 0) results = self.get('%s/results' % job_id) self.assertGreater(len(results), 0) for res in results: for etype in res['outtypes']: # test all export types if etype == 'xml' and res['type'] == 'gmf_data': continue # do not export GMFs in XML for event based text = self.get_text('result/%s' % res['id'], export_type=etype) print('downloading result/%s' % res['id'], res['type'], etype) self.assertGreater(len(text), 0) # test no filtering in actions.get_calcs all_jobs = self.get('list') self.assertGreater(len(all_jobs), 0) extract_url = '/v1/calc/%s/extract/' % job_id # check extract/composite_risk_model.attrs url = extract_url + 'composite_risk_model.attrs' self.assertEqual(self.c.get(url).status_code, 200) # check asset_tags resp = self.c.get(extract_url + 'asset_tags') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['taxonomy']), 7) # check exposure_metadata resp = self.c.get(extract_url + 'exposure_metadata') got = loadnpz(resp.streaming_content) self.assertEqual(sorted(got['tagnames']), ['id', 'taxonomy']) self.assertEqual(sorted(got['array']), ['number', 'value-structural']) # check assets resp = self.c.get(extract_url + 'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['array']), 25) # check avg_losses-rlzs resp = self.c.get(extract_url + 'agg_losses/structural?taxonomy=W-SLFB-1') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['array']), 1) # expected 1 aggregate value self.assertEqual(resp.status_code, 200) # check *-aggregation resp = self.c.get(extract_url + 'agg_losses/structural?taxonomy=*') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['tags']), 6) # expected 6 taxonomies self.assertEqual(len(got['array']), 6) # expected 6 aggregates self.assertEqual(resp.status_code, 200) # there is some logic in `core.export_from_db` that it is only # exercised when the export fails datadir, dskeys = actions.get_results(db, job_id) # try to export a non-existing output with self.assertRaises(core.DataStoreExportError) as ctx: core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp') self.assertIn('Could not export XXX in csv', str(ctx.exception)) # check MFD distribution extract_url = '/v1/calc/%s/extract/event_based_mfd' % job_id got = loadnpz(self.c.get(extract_url)) self.assertGreater(len(got['array']['mag']), 1) self.assertGreater(len(got['array']['freq']), 1)
def test_ok(self): job_id = self.postzip('archive_ok.zip') self.wait() log = self.get('%s/log/:' % job_id) self.assertGreater(len(log), 0) results = self.get('%s/results' % job_id) self.assertGreater(len(results), 0) for res in results: for etype in res['outtypes']: # test all export types text = self.get_text('result/%s' % res['id'], export_type=etype) print('downloading result/%s' % res['id'], res['type'], etype) self.assertGreater(len(text), 0) # test no filtering in actions.get_calcs all_jobs = self.get('list') self.assertGreater(len(all_jobs), 0) extract_url = '/v1/calc/%s/extract/' % job_id # check extract/composite_risk_model.attrs url = extract_url + 'composite_risk_model.attrs' self.assertEqual(self.c.get(url).status_code, 200) # check asset_tags resp = self.c.get(extract_url + 'asset_tags') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['taxonomy']), 7) # check exposure_metadata resp = self.c.get(extract_url + 'exposure_metadata') got = loadnpz(resp.streaming_content) self.assertEqual(sorted(got['tagnames']), [b'taxonomy']) self.assertEqual(sorted(got['array']), ['number', 'value-structural']) # check assets resp = self.c.get(extract_url + 'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['array']), 25) # check avg_losses-rlzs resp = self.c.get(extract_url + 'agg_losses/structural?taxonomy=W-SLFB-1') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['array']), 1) # expected 1 aggregate value self.assertEqual(resp.status_code, 200) # check *-aggregation resp = self.c.get(extract_url + 'agg_losses/structural?taxonomy=*') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['tags']), 6) # expected 6 taxonomies self.assertEqual(len(got['array']), 6) # expected 6 aggregates self.assertEqual(resp.status_code, 200) # there is some logic in `core.export_from_db` that it is only # exercised when the export fails datadir, dskeys = actions.get_results(db, job_id) # try to export a non-existing output with self.assertRaises(core.DataStoreExportError) as ctx: core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp') self.assertIn('Could not export XXX in csv', str(ctx.exception)) # check MFD distribution extract_url = '/v1/calc/%s/extract/event_based_mfd?kind=mean' % job_id got = loadnpz(self.c.get(extract_url)) self.assertGreater(len(got['magnitudes']), 1) self.assertGreater(len(got['mean_frequency']), 1) # check rupture_info extract_url = '/v1/calc/%s/extract/rupture_info' % job_id got = loadnpz(self.c.get(extract_url)) boundaries = gzip.decompress(got['boundaries']).split(b'\n') self.assertEqual(len(boundaries), 33) self.assertEqual( boundaries[0], b'POLYGON((-77.10575 18.83643, -77.11150 18.75286, -77.18793 18.75618, -77.18146 18.84064, -77.10575 18.83643))' ) self.assertEqual( boundaries[32], b'POLYGON((-77.36446 18.50400, -77.37234 18.41776, -77.38020 18.33151, -77.38806 18.24526, -77.39591 18.15902, -77.40376 18.07277, -77.41159 17.98652, -77.41942 17.90027, -77.42724 17.81402, -77.43505 17.72777, -77.52006 17.72029, -77.60506 17.71277, -77.69004 17.70522, -77.77502 17.69763, -77.76438 17.78745, -77.75372 17.87728, -77.74306 17.96710, -77.73238 18.05692, -77.72169 18.14674, -77.71099 18.23656, -77.70028 18.32638, -77.68955 18.41620, -77.67883 18.50602, -77.60023 18.50556, -77.52164 18.50508, -77.44305 18.50455, -77.36446 18.50400))' ) # check num_events extract_url = '/v1/calc/%s/extract/num_events' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(got['num_events'], 34) # check gmf_data extract_url = '/v1/calc/%s/extract/gmf_data?event_id=28' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(len(got['rlz-000']), 3) # check gmf_data with no data extract_url = '/v1/calc/%s/extract/gmf_data?event_id=0' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(len(got['rlz-000']), 0) # check extract_sources extract_url = '/v1/calc/%s/extract/sources?' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(list(got), ['wkt_gz', 'src_gz', 'array']) self.assertGreater(len(got['array']), 0)
def test_ok(self): job_id = self.postzip('archive_ok.zip') self.wait() log = self.get('%s/log/:' % job_id) self.assertGreater(len(log), 0) results = self.get('%s/results' % job_id) self.assertGreater(len(results), 0) for res in results: for etype in res['outtypes']: # test all export types text = self.get_text( 'result/%s' % res['id'], export_type=etype) print('downloading result/%s' % res['id'], res['type'], etype) self.assertGreater(len(text), 0) # test no filtering in actions.get_calcs all_jobs = self.get('list') self.assertGreater(len(all_jobs), 0) extract_url = '/v1/calc/%s/extract/' % job_id # check extract/composite_risk_model.attrs url = extract_url + 'composite_risk_model.attrs' self.assertEqual(self.c.get(url).status_code, 200) # check asset_tags resp = self.c.get(extract_url + 'asset_tags') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['taxonomy']), 7) # check exposure_metadata resp = self.c.get(extract_url + 'exposure_metadata') got = loadnpz(resp.streaming_content) self.assertEqual(sorted(got['tagnames']), [b'id', b'taxonomy']) self.assertEqual(sorted(got['array']), ['number', 'value-structural']) # check assets resp = self.c.get( extract_url + 'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['array']), 25) # check avg_losses-rlzs resp = self.c.get( extract_url + 'agg_losses/structural?taxonomy=W-SLFB-1') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['array']), 1) # expected 1 aggregate value self.assertEqual(resp.status_code, 200) # check *-aggregation resp = self.c.get( extract_url + 'agg_losses/structural?taxonomy=*') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['tags']), 6) # expected 6 taxonomies self.assertEqual(len(got['array']), 6) # expected 6 aggregates self.assertEqual(resp.status_code, 200) # there is some logic in `core.export_from_db` that it is only # exercised when the export fails datadir, dskeys = actions.get_results(db, job_id) # try to export a non-existing output with self.assertRaises(core.DataStoreExportError) as ctx: core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp') self.assertIn('Could not export XXX in csv', str(ctx.exception)) # check MFD distribution extract_url = '/v1/calc/%s/extract/event_based_mfd?kind=mean' % job_id got = loadnpz(self.c.get(extract_url)) self.assertGreater(len(got['magnitudes']), 1) self.assertGreater(len(got['mean_frequency']), 1) # check rupture_info extract_url = '/v1/calc/%s/extract/rupture_info' % job_id got = loadnpz(self.c.get(extract_url)) boundaries = gzip.decompress(got['boundaries']).split(b'\n') self.assertEqual(len(boundaries), 33) # check num_events extract_url = '/v1/calc/%s/extract/num_events' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(got['num_events'], 34) # check gmf_data extract_url = '/v1/calc/%s/extract/gmf_data?event_id=28' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(len(got['rlz-000']), 3) # check gmf_data with no data extract_url = '/v1/calc/%s/extract/gmf_data?event_id=0' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(len(got['rlz-000']), 0) # check extract_sources extract_url = '/v1/calc/%s/extract/sources?sm_id=0' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(list(got), ['sm_id', 'wkt_gz', 'src_gz', 'array']) self.assertGreater(len(got['array']), 0)
def test_ok(self): job_id = self.postzip('archive_ok.zip') self.wait() log = self.get('%s/log/:' % job_id) self.assertGreater(len(log), 0) results = self.get('%s/results' % job_id) self.assertGreater(len(results), 0) for res in results: for etype in res['outtypes']: # test all export types if etype == 'xml' and res['type'] == 'gmf_data': continue # do not export GMFs in XML for event based text = self.get_text( 'result/%s' % res['id'], export_type=etype) print('downloading result/%s' % res['id'], res['type'], etype) self.assertGreater(len(text), 0) # test no filtering in actions.get_calcs all_jobs = self.get('list') self.assertGreater(len(all_jobs), 0) extract_url = '/v1/calc/%s/extract/' % job_id # check extract/composite_risk_model.attrs url = extract_url + 'composite_risk_model.attrs' self.assertEqual(self.c.get(url).status_code, 200) # check asset_values resp = self.c.get(extract_url + 'asset_values/0') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['array']), 49) # 49 assets on site 0 self.assertEqual(resp.status_code, 200) # check asset_tags resp = self.c.get(extract_url + 'asset_tags') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['taxonomy']), 7) # check exposure_metadata resp = self.c.get(extract_url + 'exposure_metadata') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(list(got['tagnames']), ['taxonomy']) self.assertEqual(list(got['array']), ['number', 'value-structural']) # check assets resp = self.c.get( extract_url + 'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['array']), 25) # check avg_losses-rlzs resp = self.c.get( extract_url + 'agg_losses/structural?taxonomy=W-SLFB-1') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['array']), 1) # expected 1 aggregate value self.assertEqual(resp.status_code, 200) # check *-aggregation resp = self.c.get( extract_url + 'agg_losses/structural?taxonomy=*') data = b''.join(ln for ln in resp.streaming_content) got = numpy.load(io.BytesIO(data)) # load npz file self.assertEqual(len(got['tags']), 6) # expected 6 taxonomies self.assertEqual(len(got['array']), 6) # expected 6 aggregates self.assertEqual(resp.status_code, 200) # there is some logic in `core.export_from_db` that it is only # exercised when the export fails datadir, dskeys = actions.get_results(db, job_id) # try to export a non-existing output with self.assertRaises(core.DataStoreExportError) as ctx: core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp') self.assertIn('Could not export XXX in csv', str(ctx.exception)) # check MFD distribution extract_url = '/v1/calc/%s/extract/event_based_mfd' % job_id data = b''.join(ln for ln in self.c.get(extract_url)) got = numpy.load(io.BytesIO(data)) # load npz file self.assertGreater(len(got['array']['mag']), 1) self.assertGreater(len(got['array']['freq']), 1)
def test_ok(self): job_id = self.postzip('archive_ok.zip') self.wait() log = self.get('%s/log/:' % job_id) self.assertGreater(len(log), 0) results = self.get('%s/results' % job_id) self.assertGreater(len(results), 0) for res in results: for etype in res['outtypes']: # test all export types text = self.get_text('result/%s' % res['id'], export_type=etype) print('downloading result/%s' % res['id'], res['type'], etype) self.assertGreater(len(text), 0) # test no filtering in actions.get_calcs all_jobs = self.get('list') self.assertGreater(len(all_jobs), 0) extract_url = '/v1/calc/%s/extract/' % job_id # check eids_by_gsim resp = self.c.get(extract_url + 'eids_by_gsim') dic = dict(loadnpz(resp.streaming_content)) self.assertEqual(len(dic['[AtkinsonBoore2003SInter]']), 7) # check extract/composite_risk_model.attrs url = extract_url + 'composite_risk_model.attrs' self.assertEqual(self.c.get(url).status_code, 200) # check asset_tags resp = self.c.get(extract_url + 'asset_tags') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['taxonomy']), 7) # check exposure_metadata resp = self.c.get(extract_url + 'exposure_metadata') got = loadnpz(resp.streaming_content)['json'] dic = json.loads(bytes(got)) self.assertEqual(sorted(dic['tagnames']), ['taxonomy']) self.assertEqual(sorted(dic['names']), ['number', 'value-structural']) # check assets resp = self.c.get(extract_url + 'assets?taxonomy=MC-RLSB-2&taxonomy=W-SLFB-1') if resp.status_code == 500: # should never happen raise RuntimeError(resp.content.decode('utf8')) got = loadnpz(resp.streaming_content) self.assertEqual(len(got['array']), 25) # check losses_by_asset resp = self.c.get(extract_url + 'losses_by_asset') if resp.status_code == 500: # should never happen raise RuntimeError(resp.content.decode('utf8')) got = loadnpz(resp.streaming_content) self.assertEqual(len(got['rlz-000']), 95) # check agg_losses resp = self.c.get(extract_url + 'agg_losses/structural?taxonomy=W-SLFB-1') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['array']), 1) # expected 1 aggregate value self.assertEqual(resp.status_code, 200) # check *-aggregation resp = self.c.get(extract_url + 'agg_losses/structural?taxonomy=*') got = loadnpz(resp.streaming_content) self.assertEqual(len(got['tags']), 6) # expected 6 taxonomies self.assertEqual(len(got['array']), 6) # expected 6 aggregates self.assertEqual(resp.status_code, 200) # there is some logic in `core.export_from_db` that it is only # exercised when the export fails datadir, dskeys = actions.get_results(db, job_id) # try to export a non-existing output with self.assertRaises(core.DataStoreExportError) as ctx: core.export_from_db(('XXX', 'csv'), job_id, datadir, '/tmp') self.assertIn('Could not export XXX in csv', str(ctx.exception)) # check MFD distribution extract_url = '/v1/calc/%s/extract/event_based_mfd?kind=mean' % job_id got = loadnpz(self.c.get(extract_url)) self.assertGreater(len(got['magnitudes']), 1) self.assertGreater(len(got['mean_frequency']), 1) # check rupture_info extract_url = '/v1/calc/%s/extract/rupture_info' % job_id got = loadnpz(self.c.get(extract_url)) boundaries = gzip.decompress(got['boundaries']).split(b'\n') self.assertEqual(len(boundaries), 37) self.assertEqual( boundaries[0], b'POLYGON((-77.24583 17.99602, -77.25224 17.91156, -77.33583 17.90593, -77.32871 17.99129, -77.24583 17.99602))' ) self.assertEqual( boundaries[-1], b'POLYGON((-77.10000 18.92000, -77.10575 18.83643, -77.11150 18.75286, -77.11723 18.66929, -77.12297 18.58572, -77.12869 18.50215, -77.13442 18.41858, -77.14014 18.33500, -77.14584 18.25143, -77.15155 18.16786, -77.15725 18.08429, -77.16295 18.00072, -77.16864 17.91714, -77.17432 17.83357, -77.18000 17.75000, -77.26502 17.74263, -77.35004 17.73522, -77.43505 17.72777, -77.52006 17.72029, -77.60506 17.71277, -77.69004 17.70522, -77.77502 17.69763, -77.86000 17.69000, -77.84865 17.78072, -77.83729 17.87144, -77.82591 17.96215, -77.81452 18.05287, -77.80312 18.14359, -77.79172 18.23430, -77.78030 18.32502, -77.76886 18.41573, -77.75742 18.50644, -77.74596 18.59716, -77.73448 18.68787, -77.72300 18.77858, -77.71151 18.86929, -77.70000 18.96000, -77.62498 18.95510, -77.54997 18.95018, -77.47497 18.94523, -77.39996 18.94024, -77.32497 18.93523, -77.24997 18.93018, -77.17499 18.92511, -77.10000 18.92000))' ) # check num_events extract_url = '/v1/calc/%s/extract/num_events' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(got['num_events'], 41) # check gmf_data with no data extract_url = '/v1/calc/%s/extract/gmf_data?event_id=28' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(len(got['rlz-000']), 0) # check extract_sources extract_url = '/v1/calc/%s/extract/sources?' % job_id got = loadnpz(self.c.get(extract_url)) self.assertEqual(list(got), ['wkt_gz', 'src_gz', 'array']) self.assertGreater(len(got['array']), 0)