def test_calculate_school_damage(self): """Earthquake school damage calculation works via the HTTP REST API """ # Upload required data first for filename in ['lembang_mmi_hazmap.asc', 'lembang_schools.shp']: thefile = os.path.join(TESTDATA, filename) uploaded = save_to_geonode(thefile, user=self.user, overwrite=True) check_layer(uploaded, full=True) # Run calculation through API c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard='geonode:lembang_mmi_hazmap', exposure_server=INTERNAL_SERVER_URL, exposure='geonode:lembang_schools', bbox='105.592,-7.809,110.159,-5.647', impact_function='Earthquake Building Damage Function', keywords='test,schools,lembang', )) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) assert 'hazard_layer' in data.keys() assert 'exposure_layer' in data.keys() assert 'run_duration' in data.keys() assert 'run_date' in data.keys() assert 'layer' in data.keys()
def test_plugin_compatibility(self): """Default plugins perform as expected """ # Upload a raster and a vector data set hazard_filename = os.path.join(TESTDATA, 'shakemap_padang_20090930.asc') hazard_layer = save_to_geonode(hazard_filename) check_layer(hazard_layer, full=True) exposure_filename = os.path.join(TESTDATA, 'lembang_schools.shp') exposure_layer = save_to_geonode(exposure_filename) check_layer(exposure_layer, full=True) # Test plugin_list = get_plugins() assert len(plugin_list) > 0 geoserver = {'url': settings.GEOSERVER_BASE_URL + 'ows', 'name': 'Local Geoserver', 'version': '1.0.0', 'id': 0} metadata = get_layer_descriptors(geoserver['url']) msg = 'There were no layers in test geoserver' assert len(metadata) > 0, msg # Characterisation test to preserve the behaviour of # get_layer_descriptors. FIXME: I think we should change this to be # a dictionary of metadata entries (ticket #126). reference = [['geonode:lembang_schools', {'layer_type': 'feature', 'category': 'exposure', 'subcategory': 'building', 'title': 'lembang_schools'}], ['geonode:shakemap_padang_20090930', {'layer_type': 'raster', 'category': 'hazard', 'subcategory': 'earthquake', 'title': 'shakemap_padang_20090930'}]] for entry in reference: name, mdblock = entry i = [x[0] for x in metadata].index(name) assert name == metadata[i][0] for key in entry[1]: assert entry[1][key] == metadata[i][1][key] # Check plugins are returned annotated_plugins = [{'name': name, 'doc': f.__doc__, 'layers': compatible_layers(f, metadata)} for name, f in plugin_list.items()] msg = 'No compatible layers returned' assert len(annotated_plugins) > 0, msg
def test_shapefile(self): """Shapefile can be uploaded """ thefile = os.path.join(TESTDATA, 'lembang_schools.shp') layer = save_to_geonode(thefile, user=self.user, overwrite=True) check_layer(layer, full=True) assert isinstance(layer.geographic_bounding_box, basestring)
def test_calculate_fatality(self): """Earthquake fatalities calculation via the HTTP Rest API is correct """ # Upload required data first for filename in ['Earthquake_Ground_Shaking.asc', 'Population_2010_clip.tif']: thefile = os.path.join(TESTDATA, filename) uploaded = save_to_geonode(thefile, user=self.user, overwrite=True) check_layer(uploaded, full=True) # Run calculation through API c = Client() rv = c.post('/impact/api/calculate/', dict(hazard_server=INTERNAL_SERVER_URL, hazard='geonode:earthquake_ground_shaking', exposure='geonode:population_2010_clip', exposure_server=INTERNAL_SERVER_URL, bbox='99.36,-2.199,102.237,0.00', impact_function='Earthquake Fatality Function', keywords='test,earthquake,fatality')) msg = 'Expected status code 200, got %i' % rv.status_code self.assertEqual(rv.status_code, 200), msg msg = ('Expected Content-Type "application/json", ' 'got %s' % rv['Content-Type']) self.assertEqual(rv['Content-Type'], 'application/json'), msg data = json.loads(rv.content) if data['stacktrace'] is not None: msg = data['stacktrace'] raise Exception(msg) assert 'hazard_layer' in data.keys() assert 'exposure_layer' in data.keys() assert 'run_duration' in data.keys() assert 'run_date' in data.keys() assert 'layer' in data.keys() assert 'bbox' in data.keys() assert 'impact_function' in data.keys() layer_uri = data['layer'] #FIXME: This is not a good way to access the layer name typename = layer_uri.split('/')[4] name = typename.split(':')[1] # Check the autogenerated styles were correctly uploaded layer = Layer.objects.get(name=name) msg = ('A new style should have been created for layer [%s] ' 'got [%s] style instead.' % (name, layer.default_style.name)) assert layer.default_style.name == name, msg
def test_another_asc(self): """Real world ASCII file can be uploaded """ thefile = os.path.join(TESTDATA, 'lembang_mmi_hazmap.asc') layer = save_to_geonode(thefile, user=self.user, overwrite=True) check_layer(layer, full=True) # Verify metadata layer_name = '%s:%s' % (layer.workspace, layer.name) metadata = get_metadata(INTERNAL_SERVER_URL, layer_name) assert 'id' in metadata assert 'title' in metadata assert 'layer_type' in metadata assert 'keywords' in metadata assert 'bounding_box' in metadata assert 'geotransform' in metadata assert len(metadata['bounding_box']) == 4 # A little metadata characterisation test # FIXME (Ole): Get this right when new resolution keyword # has been fully sorted out. There are 3 other tests failing at # the moment ref = { 'layer_type': 'raster', 'keywords': { 'category': 'hazard', 'subcategory': 'earthquake', 'resolution': '0.0112' }, 'geotransform': (105.29857, 0.0112, 0.0, -5.565233000000001, 0.0, -0.0112), 'resolution': 0.0112, 'title': 'lembang_mmi_hazmap' } for key in ['layer_type', 'keywords', 'geotransform', 'title']: if key == 'keywords': kwds = metadata[key] for k in kwds: assert kwds[k] == ref[key][k] else: msg = ('Expected metadata for key %s to be %s. ' 'Instead got %s' % (key, ref[key], metadata[key])) if key in ['geotransform', 'resolution']: assert numpy.allclose(metadata[key], ref[key]), msg else: assert metadata[key] == ref[key], msg
def test_plugin_selection(self): """Verify the plugins can recognize compatible layers. """ # Upload a raster and a vector data set hazard_filename = os.path.join(TESTDATA, 'Earthquake_Ground_Shaking.asc') hazard_layer = save_to_geonode(hazard_filename, user=self.user, overwrite=True) check_layer(hazard_layer, full=True) msg = 'No keywords found in layer %s' % hazard_layer.name assert len(hazard_layer.keywords) > 0, msg exposure_filename = os.path.join(TESTDATA, 'lembang_schools.shp') exposure_layer = save_to_geonode(exposure_filename) check_layer(exposure_layer, full=True) msg = 'No keywords found in layer %s' % exposure_layer.name assert len(exposure_layer.keywords) > 0, msg c = Client() rv = c.post('/impact/api/functions/', data={}) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) assert 'functions' in data functions = data['functions'] # FIXME (Ariel): This test should implement an alternative function to # parse the requirements, but for now it will just take the buildings # damage one. for function in functions: if function['name'] == 'Earthquake Building Damage Function': layers = function['layers'] msg_tmpl = 'Expected layer %s in list of compatible layers: %s' hazard_msg = msg_tmpl % (hazard_layer.typename, layers) assert hazard_layer.typename in layers, hazard_msg exposure_msg = msg_tmpl % (exposure_layer.typename, layers) assert exposure_layer.typename in layers, exposure_msg
def test_cleanup(self): """Cleanup functions in the utils module work """ from geonode.maps.utils import cleanup thefile = os.path.join(TESTDATA, 'lembang_mmi_hazmap.asc') uploaded = save_to_geonode(thefile, user=self.user, overwrite=True) check_layer(uploaded, full=True) name = uploaded.name uuid = uploaded.uuid pk = uploaded.pk # try calling the cleanup function when the django record exists: try: cleanup(name, uuid) except GeoNodeException, e: pass
def test_another_asc(self): """Real world ASCII file can be uploaded """ thefile = os.path.join(TESTDATA, 'lembang_mmi_hazmap.asc') layer = save_to_geonode(thefile, user=self.user, overwrite=True) check_layer(layer, full=True) # Verify metadata layer_name = '%s:%s' % (layer.workspace, layer.name) metadata = get_metadata(INTERNAL_SERVER_URL, layer_name) assert 'id' in metadata assert 'title' in metadata assert 'layer_type' in metadata assert 'keywords' in metadata assert 'bounding_box' in metadata assert 'geotransform' in metadata assert len(metadata['bounding_box']) == 4 # A little metadata characterisation test ref = {'layer_type': 'raster', 'keywords': {'category': 'hazard', 'subcategory': 'earthquake'}, 'geotransform': (105.29857, 0.0112, 0.0, -5.565233000000001, 0.0, -0.0112), 'title': 'lembang_mmi_hazmap'} for key in ['layer_type', 'keywords', 'geotransform', 'title']: msg = ('Expected metadata for key %s to be %s. ' 'Instead got %s' % (key, ref[key], metadata[key])) if key == 'geotransform': assert numpy.allclose(metadata[key], ref[key]), msg else: assert metadata[key] == ref[key], msg if key == 'keywords': kwds = metadata[key] for k in kwds: assert kwds[k] == ref[key][k]
def test_calculate_school_damage(self): """Earthquake school damage calculation works via the HTTP REST API """ # Upload required data first for filename in ['lembang_mmi_hazmap.asc', 'lembang_schools.shp']: thefile = os.path.join(TESTDATA, filename) uploaded = save_to_geonode(thefile, user=self.user, overwrite=True) check_layer(uploaded, full=True) # Run calculation through API c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard='geonode:lembang_mmi_hazmap', exposure_server=INTERNAL_SERVER_URL, exposure='geonode:lembang_schools', bbox='105.592,-7.809,110.159,-5.647', impact_function='Earthquake Building Damage Function', keywords='test,schools,lembang', )) msg = 'Expected status code 200, got %i' % rv.status_code self.assertEqual(rv.status_code, 200), msg msg = ('Expected Content-Type "application/json", ' 'got %s' % rv['Content-Type']) self.assertEqual(rv['Content-Type'], 'application/json'), msg data = json.loads(rv.content) if data['stacktrace'] is not None: msg = data['stacktrace'] raise Exception(msg) assert 'hazard_layer' in data.keys() assert 'exposure_layer' in data.keys() assert 'run_duration' in data.keys() assert 'run_date' in data.keys() assert 'layer' in data.keys()
def test_repeated_upload(self): """The same file can be uploaded more than once """ thefile = os.path.join(TESTDATA, 'test_grid.asc') uploaded1 = save_to_geonode(thefile, overwrite=True, user=self.user) check_layer(uploaded1, full=True) uploaded2 = save_to_geonode(thefile, overwrite=True, user=self.user) check_layer(uploaded2, full=True) uploaded3 = save_to_geonode(thefile, overwrite=False, user=self.user) check_layer(uploaded3, full=True) msg = ('Expected %s but got %s' % (uploaded1.name, uploaded2.name)) assert uploaded1.name == uploaded2.name, msg msg = ('Expected a different name when uploading %s using ' 'overwrite=False but got %s' % (thefile, uploaded3.name)) assert uploaded1.name != uploaded3.name, msg
def test_asc(self): """ASCII file can be uploaded """ thefile = os.path.join(TESTDATA, 'test_grid.asc') uploaded = save_to_geonode(thefile, user=self.user, overwrite=True) check_layer(uploaded, full=True)
def test_tiff(self): """GeoTIF file can be uploaded """ thefile = os.path.join(TESTDATA, 'Population_2010_clip.tif') uploaded = save_to_geonode(thefile, user=self.user, overwrite=True) check_layer(uploaded, full=True)
def test_plugin_compatibility(self): """Default plugins perform as expected """ # Upload a raster and a vector data set hazard_filename = os.path.join(TESTDATA, 'shakemap_padang_20090930.asc') hazard_layer = save_to_geonode(hazard_filename) check_layer(hazard_layer, full=True) exposure_filename = os.path.join(TESTDATA, 'lembang_schools.shp') exposure_layer = save_to_geonode(exposure_filename) check_layer(exposure_layer, full=True) # Test plugin_list = get_plugins() assert len(plugin_list) > 0 geoserver = { 'url': settings.GEOSERVER_BASE_URL + 'ows', 'name': 'Local Geoserver', 'version': '1.0.0', 'id': 0 } metadata = get_layer_descriptors(geoserver['url']) msg = 'There were no layers in test geoserver' assert len(metadata) > 0, msg # Characterisation test to preserve the behaviour of # get_layer_descriptors. FIXME: I think we should change this to be # a dictionary of metadata entries (ticket #126). reference = [[ 'geonode:lembang_schools', { 'layer_type': 'vector', 'category': 'exposure', 'subcategory': 'building', 'title': 'lembang_schools' } ], [ 'geonode:shakemap_padang_20090930', { 'layer_type': 'raster', 'category': 'hazard', 'subcategory': 'earthquake', 'title': 'shakemap_padang_20090930' } ]] for entry in reference: name, mdblock = entry i = [x[0] for x in metadata].index(name) msg = 'Got name %s, expected %s' % (name, metadata[i][0]) assert name == metadata[i][0], msg for key in entry[1]: refval = entry[1][key] val = metadata[i][1][key] msg = ('Got value "%s" for key "%s" ' 'Expected "%s"' % (val, key, refval)) assert refval == val, msg # Check plugins are returned annotated_plugins = [{ 'name': name, 'doc': f.__doc__, 'layers': compatible_layers(f, metadata) } for name, f in plugin_list.items()] msg = 'No compatible layers returned' assert len(annotated_plugins) > 0, msg
def test_raster_wcs_reprojection(self): """UTM Raster can be reprojected by Geoserver and downloaded correctly """ # FIXME (Ole): Still need to do this with assertions filename = 'tsunami_max_inundation_depth_BB_utm.asc' projected_tif_file = os.path.join(TESTDATA, filename) #projected_tif = file_upload(projected_tif_file, overwrite=True) projected_tif = save_to_geonode(projected_tif_file, user=self.user, overwrite=True) check_layer(projected_tif) wcs_url = settings.GEOSERVER_BASE_URL + 'wcs' wcs = WebCoverageService(wcs_url, version='1.0.0') #logger.info(wcs.contents) metadata = wcs.contents[projected_tif.typename] #logger.info(metadata.grid) bboxWGS84 = metadata.boundingBoxWGS84 #logger.info(bboxWGS84) resx = metadata.grid.offsetvectors[0][0] resy = abs(float(metadata.grid.offsetvectors[1][1])) #logger.info("resx=%s resy=%s" % (str(resx), str(resy))) formats = metadata.supportedFormats #logger.info(formats) supportedCRS = metadata.supportedCRS #logger.info(supportedCRS) width = metadata.grid.highlimits[0] height = metadata.grid.highlimits[1] #logger.info("width=%s height=%s" % (width, height)) gs_cat = Layer.objects.gs_catalog cvg_store = gs_cat.get_store(projected_tif.name) cvg_layer = gs_cat.get_resource(projected_tif.name, store=cvg_store) #logger.info(cvg_layer.request_srs_list) #logger.info(cvg_layer.response_srs_list) # FIXME: A patch was submitted OWSlib 20110808 # Can delete the following once patch appears # In the future get bboxNative and nativeSRS from get_metadata descCov = metadata._service.getDescribeCoverage(projected_tif.typename) envelope = (descCov.find(ns('CoverageOffering/') + ns('domainSet/') + ns('spatialDomain/') + '{http://www.opengis.net/gml}Envelope')) nativeSrs = envelope.attrib['srsName'] #logger.info(nativeSrs) gmlpositions = envelope.findall('{http://www.opengis.net/gml}pos') lc = gmlpositions[0].text uc = gmlpositions[1].text bboxNative = (float(lc.split()[0]), float(lc.split()[1]), float(uc.split()[0]), float(uc.split()[1])) #logger.info(bboxNative) # ---- END PATCH # Make a temp dir to store the saved files tempdir = '/tmp/%s' % str(time.time()) os.mkdir(tempdir) # Check that the layer can be downloaded in its native projection cvg = wcs.getCoverage(identifier=projected_tif.typename, format='GeoTIFF', crs=nativeSrs, bbox=bboxNative, resx=resx, resy=resy) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close() #logger.info("GeoTIFF in %s = %s" % (nativeSrs, t.name)) # TODO: Verify that the file is a valid GeoTiff and that it is # _exactly_ the same size and bbox of the original # Test that the layer can be downloaded in ARCGRID format cvg_layer.supported_formats = cvg_layer.supported_formats + ['ARCGRID'] gs_cat.save(cvg_layer) cvg = wcs.getCoverage(identifier=projected_tif.typename, format='ARCGRID', crs=nativeSrs, bbox=bboxNative, resx=resx, resy=resy) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close() #logger.info("ARCGRID in %s = %s" % (nativeSrs, t.name)) # Check that the downloaded file is a valid ARCGRID file and that it # the required projection information # (FIXME: There is no prj file here. GS bug) # Check that the layer can downloaded in WGS84 cvg_layer.request_srs_list += ['EPSG:4326'] cvg_layer.response_srs_list += ['EPSG:4326'] gs_cat.save(cvg_layer) #logger.info(cvg_layer.request_srs_list) #logger.info(cvg_layer.response_srs_list) cvg = wcs.getCoverage(identifier=projected_tif.typename, format='GeoTIFF', crs='EPSG:4326', bbox=bboxWGS84, #resx=0.000202220898116, # Should NOT be hard-coded! # How do we convert #resy=0.000202220898116) # See comments in riab issue #103 width=width, height=height) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close() #logger.info("GeoTIFF in %s = %s" % ("EPSG:4326", t.name)) # TODO: Verify that the file is a valid GeoTiff and that it is # the correct size and bbox based on the resx and resy or width # and height specified # Check that we can download the layer in another projection cvg_layer.request_srs_list += ['EPSG:32356'] cvg_layer.response_srs_list += ['EPSG:32356'] cvg_layer.request_srs_list += ['EPSG:900913'] cvg_layer.response_srs_list += ['EPSG:900913'] gs_cat.save(cvg_layer) #logger.info(cvg_layer.request_srs_list) #logger.info(cvg_layer.response_srs_list) # How do we get the bboxes for the newly assigned # request/response SRS?? cvg = wcs.getCoverage(identifier=projected_tif.typename, format='GeoTIFF', crs='EPSG:32356', # Should not be hardcoded for a test, # or should use 900913 (need bbox) bbox=bboxNative, #resx=0.000202220898116, # Should NOT be hard-coded! # How do we convert #resy=0.000202220898116) # See comments in riab issue #103 width=width, height=height) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close()
def test_linked_datasets(self): """Linked datesets can be pulled in e.g. to include gender break down """ # Upload exposure data for this test. This will automatically # pull in female_pct_yogya.asc through its "associates" keyword name = 'population_yogya' exposure_filename = '%s/%s.asc' % (TESTDATA, name) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) exposure_name = '%s:%s' % (exposure_layer.workspace, exposure_layer.name) # Check metadata assert_bounding_box_matches(exposure_layer, exposure_filename) exp_bbox_string = get_bounding_box_string(exposure_filename) check_layer(exposure_layer, full=True) # Upload hazard data filename = 'eq_yogya_2006.asc' hazard_filename = '%s/%s' % (TESTDATA, filename) hazard_layer = save_to_geonode(hazard_filename, user=self.user, overwrite=True) hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name) # Check metadata assert_bounding_box_matches(hazard_layer, hazard_filename) haz_bbox_string = get_bounding_box_string(hazard_filename) check_layer(hazard_layer, full=True) # Run calculation c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard=hazard_name, exposure_server=INTERNAL_SERVER_URL, exposure=exposure_name, bbox=haz_bbox_string, impact_function='EarthquakeFatalityFunction', keywords='test,fatalities,population,usgs')) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) if 'errors' in data: errors = data['errors'] if errors is not None: msg = ('The server returned the error message: %s' % str(errors)) raise Exception(msg) assert 'success' in data assert 'hazard_layer' in data assert 'exposure_layer' in data assert 'run_duration' in data assert 'run_date' in data assert 'layer' in data assert data['success'] # Download result and check layer_name = data['layer'].split('/')[-1] result_layer = download(INTERNAL_SERVER_URL, layer_name, get_bounding_box_string(hazard_filename)) assert os.path.exists(result_layer.filename) # Check calculated values keywords = result_layer.get_keywords() assert 'caption' in keywords
def test_earthquake_exposure_plugin(self): """Population exposure to individual MMI levels can be computed """ # Upload exposure data for this test # FIXME (Ole): While this dataset is ok for testing, # note that is has been resampled without scaling # so numbers are about 25 times too large. # Consider replacing test populations dataset for good measures, # just in case any one accidentally started using this dataset # for real. name = 'Population_2010' exposure_filename = '%s/%s.asc' % (TESTDATA, name) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) exposure_name = '%s:%s' % (exposure_layer.workspace, exposure_layer.name) # Check metadata assert_bounding_box_matches(exposure_layer, exposure_filename) exp_bbox_string = get_bounding_box_string(exposure_filename) check_layer(exposure_layer, full=True) # Upload hazard data filename = 'Lembang_Earthquake_Scenario.asc' hazard_filename = '%s/%s' % (TESTDATA, filename) hazard_layer = save_to_geonode(hazard_filename, user=self.user, overwrite=True) hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name) # Check metadata assert_bounding_box_matches(hazard_layer, hazard_filename) haz_bbox_string = get_bounding_box_string(hazard_filename) check_layer(hazard_layer, full=True) # Run calculation c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard=hazard_name, exposure_server=INTERNAL_SERVER_URL, exposure=exposure_name, bbox=haz_bbox_string, impact_function='EarthquakePopulationExposureFunction', keywords='test,population,exposure,usgs')) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) if 'errors' in data: errors = data['errors'] if errors is not None: msg = ('The server returned the error message: %s' % str(errors)) raise Exception(msg) assert 'success' in data assert 'hazard_layer' in data assert 'exposure_layer' in data assert 'run_duration' in data assert 'run_date' in data assert 'layer' in data assert data['success'] # Download result and check layer_name = data['layer'].split('/')[-1] result_layer = download(INTERNAL_SERVER_URL, layer_name, get_bounding_box_string(hazard_filename)) assert os.path.exists(result_layer.filename) # Check calculated values keywords = result_layer.get_keywords() assert 'mmi-classes' in keywords assert 'affected-population' in keywords mmi_classes = [int(x) for x in keywords['mmi-classes'].split('_')] count = [float(x) for x in keywords['affected-population'].split('_')] # Brute force count for each population level population = download(INTERNAL_SERVER_URL, exposure_name, get_bounding_box_string(hazard_filename)) intensity = download(INTERNAL_SERVER_URL, hazard_name, get_bounding_box_string(hazard_filename)) # Extract data H = intensity.get_data(nan=0) P = population.get_data(nan=0) brutecount = {} for mmi in mmi_classes: brutecount[mmi] = 0 for i in range(P.shape[0]): for j in range(P.shape[1]): mmi = H[i, j] if not numpy.isnan(mmi): mmi_class = int(round(mmi)) pop = P[i, j] if not numpy.isnan(pop): brutecount[mmi_class] += pop for i, mmi in enumerate(mmi_classes): assert numpy.allclose(count[i], brutecount[mmi], rtol=1.0e-6)
def test_raster_wcs_reprojection(self): """UTM Raster can be reprojected by Geoserver and downloaded correctly """ # FIXME (Ole): Jeff needs to do this with assertions (ticket #40) filename = 'tsunami_max_inundation_depth_BB_utm.asc' projected_tif_file = os.path.join(TESTDATA, filename) #projected_tif = file_upload(projected_tif_file, overwrite=True) projected_tif = save_to_geonode(projected_tif_file, user=self.user, overwrite=True) check_layer(projected_tif) wcs_url = settings.GEOSERVER_BASE_URL + 'wcs' wcs = WebCoverageService(wcs_url, version='1.0.0') #logger.info(wcs.contents) metadata = wcs.contents[projected_tif.typename] #logger.info(metadata.grid) bboxWGS84 = metadata.boundingBoxWGS84 #logger.info(bboxWGS84) resx = metadata.grid.offsetvectors[0][0] resy = abs(float(metadata.grid.offsetvectors[1][1])) #logger.info("resx=%s resy=%s" % (str(resx), str(resy))) formats = metadata.supportedFormats #logger.info(formats) supportedCRS = metadata.supportedCRS #logger.info(supportedCRS) width = metadata.grid.highlimits[0] height = metadata.grid.highlimits[1] #logger.info("width=%s height=%s" % (width, height)) gs_cat = Layer.objects.gs_catalog cvg_store = gs_cat.get_store(projected_tif.name) cvg_layer = gs_cat.get_resource(projected_tif.name, store=cvg_store) #logger.info(cvg_layer.request_srs_list) #logger.info(cvg_layer.response_srs_list) # FIXME: A patch was submitted OWSlib 20110808 # Can delete the following once patch appears # In the future get bboxNative and nativeSRS from get_metadata descCov = metadata._service.getDescribeCoverage(projected_tif.typename) envelope = (descCov.find( ns('CoverageOffering/') + ns('domainSet/') + ns('spatialDomain/') + '{http://www.opengis.net/gml}Envelope')) nativeSrs = envelope.attrib['srsName'] #logger.info(nativeSrs) gmlpositions = envelope.findall('{http://www.opengis.net/gml}pos') lc = gmlpositions[0].text uc = gmlpositions[1].text bboxNative = (float(lc.split()[0]), float(lc.split()[1]), float(uc.split()[0]), float(uc.split()[1])) #logger.info(bboxNative) # ---- END PATCH # Make a temp dir to store the saved files tempdir = '/tmp/%s' % str(time.time()) os.mkdir(tempdir) # Check that the layer can be downloaded in its native projection cvg = wcs.getCoverage(identifier=projected_tif.typename, format='GeoTIFF', crs=nativeSrs, bbox=bboxNative, resx=resx, resy=resy) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close() #logger.info("GeoTIFF in %s = %s" % (nativeSrs, t.name)) # TODO: Verify that the file is a valid GeoTiff and that it is # _exactly_ the same size and bbox of the original # Test that the layer can be downloaded in ARCGRID format cvg_layer.supported_formats = cvg_layer.supported_formats + ['ARCGRID'] gs_cat.save(cvg_layer) cvg = wcs.getCoverage(identifier=projected_tif.typename, format='ARCGRID', crs=nativeSrs, bbox=bboxNative, resx=resx, resy=resy) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close() #logger.info("ARCGRID in %s = %s" % (nativeSrs, t.name)) # Check that the downloaded file is a valid ARCGRID file and that it # the required projection information # (FIXME: There is no prj file here. GS bug) # Check that the layer can downloaded in WGS84 cvg_layer.request_srs_list += ['EPSG:4326'] cvg_layer.response_srs_list += ['EPSG:4326'] gs_cat.save(cvg_layer) #logger.info(cvg_layer.request_srs_list) #logger.info(cvg_layer.response_srs_list) cvg = wcs.getCoverage( identifier=projected_tif.typename, format='GeoTIFF', crs='EPSG:4326', bbox=bboxWGS84, #resx=0.000202220898116, # Should NOT be hard-coded! # How do we convert #resy=0.000202220898116) # See comments in riab issue #103 width=width, height=height) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close() #logger.info("GeoTIFF in %s = %s" % ("EPSG:4326", t.name)) # TODO: Verify that the file is a valid GeoTiff and that it is # the correct size and bbox based on the resx and resy or width # and height specified # Check that we can download the layer in another projection cvg_layer.request_srs_list += ['EPSG:32356'] cvg_layer.response_srs_list += ['EPSG:32356'] cvg_layer.request_srs_list += ['EPSG:900913'] cvg_layer.response_srs_list += ['EPSG:900913'] gs_cat.save(cvg_layer) #logger.info(cvg_layer.request_srs_list) #logger.info(cvg_layer.response_srs_list) # How do we get the bboxes for the newly assigned # request/response SRS?? cvg = wcs.getCoverage( identifier=projected_tif.typename, format='GeoTIFF', crs='EPSG:32356', # Should not be hardcoded for a test, # or should use 900913 (need bbox) bbox=bboxNative, #resx=0.000202220898116, # Should NOT be hard-coded! # How do we convert #resy=0.000202220898116) # See comments in riab issue #103 width=width, height=height) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close()
def test_the_earthquake_fatality_estimation_allen(self): """Fatality computation computed correctly with GeoServer Data """ # Simulate bounding box from application viewport_bbox_string = '104.3,-8.2,110.04,-5.17' # Upload exposure data for this test name = 'Population_2010' exposure_filename = '%s/%s.asc' % (TESTDATA, name) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) workspace = exposure_layer.workspace msg = 'Expected workspace to be "geonode". Got %s' % workspace assert workspace == 'geonode' layer_name = exposure_layer.name msg = 'Expected layer name to be "%s". Got %s' % (name, layer_name) assert layer_name == name.lower(), msg exposure_name = '%s:%s' % (workspace, layer_name) # Check metadata assert_bounding_box_matches(exposure_layer, exposure_filename) exp_bbox_string = get_bounding_box_string(exposure_filename) check_layer(exposure_layer, full=True) # Now we know that exposure layer is good, lets upload some # hazard layers and do the calculations filename = 'Lembang_Earthquake_Scenario.asc' # Save hazard_filename = '%s/%s' % (TESTDATA, filename) hazard_layer = save_to_geonode(hazard_filename, user=self.user, overwrite=True) hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name) # Check metadata assert_bounding_box_matches(hazard_layer, hazard_filename) haz_bbox_string = get_bounding_box_string(hazard_filename) check_layer(hazard_layer, full=True) # Run calculation c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard=hazard_name, exposure_server=INTERNAL_SERVER_URL, exposure=exposure_name, #bbox=viewport_bbox_string, bbox=exp_bbox_string, # This one reproduced the # crash for lembang impact_function='EarthquakeFatalityFunction', keywords='test,shakemap,usgs')) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) if 'errors' in data: errors = data['errors'] if errors is not None: msg = ('The server returned the error message: %s' % str(errors)) raise Exception(msg) assert 'success' in data assert 'hazard_layer' in data assert 'exposure_layer' in data assert 'run_duration' in data assert 'run_date' in data assert 'layer' in data assert data['success'] # Download result and check layer_name = data['layer'].split('/')[-1] result_layer = download(INTERNAL_SERVER_URL, layer_name, get_bounding_box_string(hazard_filename)) assert os.path.exists(result_layer.filename)
def test_jakarta_flood_study(self): """HKV Jakarta flood study calculated correctly using the API """ # FIXME (Ole): Redo with population as shapefile later # Expected values from HKV expected_values = [2485442, 1537920] # Name files for hazard level, exposure and expected fatalities population = 'Population_Jakarta_geographic' plugin_name = 'FloodImpactFunction' # Upload exposure data for this test exposure_filename = '%s/%s.asc' % (TESTDATA, population) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) workspace = exposure_layer.workspace msg = 'Expected workspace to be "geonode". Got %s' % workspace assert workspace == 'geonode' layer_name = exposure_layer.name msg = 'Expected layer name to be "%s". Got %s' % (population, layer_name) assert layer_name.lower() == population.lower(), msg exposure_name = '%s:%s' % (workspace, layer_name) # Check metadata assert_bounding_box_matches(exposure_layer, exposure_filename) exp_bbox_string = get_bounding_box_string(exposure_filename) check_layer(exposure_layer, full=True) # Now we know that exposure layer is good, lets upload some # hazard layers and do the calculations i = 0 for filename in ['Flood_Current_Depth_Jakarta_geographic.asc', 'Flood_Design_Depth_Jakarta_geographic.asc']: hazard_filename = os.path.join(TESTDATA, filename) exposure_filename = os.path.join(TESTDATA, population) # Save hazard_filename = '%s/%s' % (TESTDATA, filename) hazard_layer = save_to_geonode(hazard_filename, user=self.user, overwrite=True) hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name) # Check metadata assert_bounding_box_matches(hazard_layer, hazard_filename) haz_bbox_string = get_bounding_box_string(hazard_filename) check_layer(hazard_layer, full=True) # Run calculation c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard=hazard_name, exposure_server=INTERNAL_SERVER_URL, exposure=exposure_name, bbox=exp_bbox_string, impact_function=plugin_name, keywords='test,flood,HKV')) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) if 'errors' in data: errors = data['errors'] if errors is not None: raise Exception(errors) assert 'hazard_layer' in data assert 'exposure_layer' in data assert 'run_duration' in data assert 'run_date' in data assert 'layer' in data # Do calculation manually and check result hazard_raster = read_layer(hazard_filename) H = hazard_raster.get_data(nan=0) exposure_raster = read_layer(exposure_filename + '.asc') P = exposure_raster.get_data(nan=0) # Calculate impact manually pixel_area = 2500 I = numpy.where(H > 0.1, P, 0) / 100000.0 * pixel_area # Verify correctness against results from HKV res = sum(I.flat) ref = expected_values[i] #print filename, 'Result=%f' % res, ' Expected=%f' % ref #print 'Pct relative error=%f' % (abs(res-ref)*100./ref) msg = 'Got result %f but expected %f' % (res, ref) assert numpy.allclose(res, ref, rtol=1.0e-2), msg # Verify correctness of result # Download result and check layer_name = data['layer'].split('/')[-1] result_layer = download(INTERNAL_SERVER_URL, layer_name, get_bounding_box_string(hazard_filename)) assert os.path.exists(result_layer.filename) calculated_raster = read_layer(result_layer.filename) C = calculated_raster.get_data(nan=0) # FIXME (Ole): Bring this back # Check caption #caption = calculated_raster.get_caption() #print #print caption #expct = 'people' #msg = ('Caption %s did not contain expected ' # 'keyword %s' % (caption, expct)) #assert expct in caption, msg # Compare shape and extrema msg = ('Shape of calculated raster differs from reference raster: ' 'C=%s, I=%s' % (C.shape, I.shape)) assert numpy.allclose(C.shape, I.shape, rtol=1e-12, atol=1e-12), msg msg = ('Minimum of calculated raster differs from reference ' 'raster: ' 'C=%s, I=%s' % (numpy.nanmin(C), numpy.nanmin(I))) assert numpy.allclose(numpy.nanmin(C), numpy.nanmin(I), rtol=1e-6, atol=1e-12), msg msg = ('Maximum of calculated raster differs from reference ' 'raster: ' 'C=%s, I=%s' % (numpy.nanmax(C), numpy.nanmax(I))) assert numpy.allclose(numpy.nanmax(C), numpy.nanmax(I), rtol=1e-6, atol=1e-12), msg # Compare every single value numerically (a bit loose - # probably due to single precision conversions when # data flows through geonode) # # FIXME: Not working - but since this test is about # issue #162 we'll leave it for now. TODO with NAN # Manually verified that the two expected values are correct, # though. #msg = 'Array values of written raster array were not as expected' #print C #print I #print numpy.amax(numpy.abs(C-I)) #assert numpy.allclose(C, I, rtol=1e-2, atol=1e-5), msg # Check that extrema are in range xmin, xmax = calculated_raster.get_extrema() assert numpy.alltrue(C[-numpy.isnan(C)] >= xmin), msg assert numpy.alltrue(C[-numpy.isnan(C)] <= xmax) assert numpy.alltrue(C[-numpy.isnan(C)] >= 0) i += 1