def test_io(self): """Data can be uploaded and downloaded from internal GeoServer """ # Upload a raster and a vector data set for filename in ['population_padang_1.asc', 'lembang_schools.shp']: basename, ext = os.path.splitext(filename) filename = os.path.join(TESTDATA, filename) layer = save_to_geonode(filename, user=self.user, overwrite=True) # Name checking layer_name = layer.name expected_name = basename.lower() msg = 'Expected layername %s but got %s' % (expected_name, layer_name) assert layer_name == expected_name, msg workspace = layer.workspace msg = 'Expected workspace to be "geonode". Got %s' % workspace assert workspace == 'geonode' # Check metadata assert_bounding_box_matches(layer, filename) # Download layer again using workspace:name bbox = get_bounding_box(filename) downloaded_layer = download(INTERNAL_SERVER_URL, '%s:%s' % (workspace, layer_name), bbox) assert os.path.exists(downloaded_layer.filename)
def test_metadata(self): """Metadata is retrieved correctly for both raster and vector data """ # Upload test data filenames = ['Lembang_Earthquake_Scenario.asc', 'Earthquake_Ground_Shaking.asc', 'lembang_schools.shp', 'Padang_WGS84.shp'] layers = [] paths = [] for filename in filenames: basename, ext = os.path.splitext(filename) path = os.path.join(TESTDATA, filename) layer = save_to_geonode(path, user=self.user, overwrite=True) # Record layer and file layers.append(layer) paths.append(path) # Check integrity for i, layer in enumerate(layers): if filenames[i].endswith('.shp'): layer_type = 'vector' elif filenames[i].endswith('.asc'): layer_type = 'raster' else: msg = ('Unknown layer extension in %s. ' 'Expected .shp or .asc' % filenames[i]) raise Exception(msg) layer_name = '%s:%s' % (layer.workspace, layer.name) metadata = get_metadata(INTERNAL_SERVER_URL, layer_name) assert 'id' in metadata assert 'title' in metadata assert 'layer_type' in metadata assert 'keywords' in metadata assert 'bounding_box' in metadata assert len(metadata['bounding_box']) == 4 # Check integrity between Django layer and file assert_bounding_box_matches(layer, paths[i]) # Check integrity between file and OWS metadata ref_bbox = get_bounding_box(paths[i]) msg = ('Bounding box from OWS did not match bounding box ' 'from file. They are\n' 'From file %s: %s\n' 'From OWS: %s' % (paths[i], ref_bbox, metadata['bounding_box'])) assert numpy.allclose(metadata['bounding_box'], ref_bbox), msg assert layer.name == metadata['title'] assert layer_name == metadata['id'] assert layer_type == metadata['layer_type'] # Check keywords if layer_type == 'raster': category = 'hazard' subcategory = 'earthquake' elif layer_type == 'vector': category = 'exposure' subcategory = 'building' else: msg = 'Unknown layer type %s' % layer_type raise Exception(msg) keywords = metadata['keywords'] msg = 'Did not find key "category" in keywords: %s' % keywords assert 'category' in keywords, msg msg = 'Did not find key "subcategory" in keywords: %s' % keywords assert 'subcategory' in keywords, msg msg = ('Category keyword %s did not match expected %s' % (keywords['category'], category)) assert category == keywords['category'], msg msg = ('Subcategory keyword %s did not match expected %s' % (keywords['subcategory'], category)) assert subcategory == keywords['subcategory'], msg
def test_metadata_twice(self): """Layer metadata can be correctly uploaded multiple times """ # This test reproduces ticket #99 by creating new data, # uploading twice and verifying metadata # Base test data filenames = ['Lembang_Earthquake_Scenario.asc', 'lembang_schools.shp'] for org_filename in filenames: org_basename, ext = os.path.splitext(os.path.join(TESTDATA, org_filename)) # Copy data to temporary unique name basename = unique_filename(dir='/tmp') cmd = '/bin/cp %s.keywords %s.keywords' % (org_basename, basename) os.system(cmd) cmd = '/bin/cp %s.prj %s.prj' % (org_basename, basename) os.system(cmd) if ext == '.asc': layer_type = 'raster' filename = '%s.asc' % basename cmd = '/bin/cp %s.asc %s' % (org_basename, filename) os.system(cmd) elif ext == '.shp': layer_type = 'vector' filename = '%s.shp' % basename for e in ['shp', 'shx', 'sbx', 'sbn', 'dbf']: cmd = '/bin/cp %s.%s %s.%s' % (org_basename, e, basename, e) os.system(cmd) else: msg = ('Unknown layer extension in %s. ' 'Expected .shp or .asc' % filename) raise Exception(msg) # Repeat multiple times for i in range(3): # Upload layer = save_to_geonode(filename, user=self.user, overwrite=True) # Get metadata layer_name = '%s:%s' % (layer.workspace, layer.name) metadata = get_metadata(INTERNAL_SERVER_URL, layer_name) # Verify assert 'id' in metadata assert 'title' in metadata assert 'layer_type' in metadata assert 'keywords' in metadata assert 'bounding_box' in metadata assert len(metadata['bounding_box']) == 4 # Check integrity between Django layer and file assert_bounding_box_matches(layer, filename) # Check integrity between file and OWS metadata ref_bbox = get_bounding_box(filename) msg = ('Bounding box from OWS did not match bounding box ' 'from file. They are\n' 'From file %s: %s\n' 'From OWS: %s' % (filename, ref_bbox, metadata['bounding_box'])) assert numpy.allclose(metadata['bounding_box'], ref_bbox), msg assert layer.name == metadata['title'] assert layer_name == metadata['id'] assert layer_type == metadata['layer_type'] # Check keywords if layer_type == 'raster': category = 'hazard' subcategory = 'earthquake' elif layer_type == 'vector': category = 'exposure' subcategory = 'building' else: msg = 'Unknown layer type %s' % layer_type raise Exception(msg) keywords = metadata['keywords'] msg = 'Did not find key "category" in keywords: %s' % keywords assert 'category' in keywords, msg msg = ('Did not find key "subcategory" in keywords: %s' % keywords) assert 'subcategory' in keywords, msg msg = ('Category keyword %s did not match expected %s' % (keywords['category'], category)) assert category == keywords['category'], msg msg = ('Subcategory keyword %s did not match expected %s' % (keywords['subcategory'], category)) assert subcategory == keywords['subcategory'], msg
def test_linked_datasets(self): """Linked datesets can be pulled in e.g. to include gender break down """ # Upload exposure data for this test. This will automatically # pull in female_pct_yogya.asc through its "associates" keyword name = 'population_yogya' exposure_filename = '%s/%s.asc' % (TESTDATA, name) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) exposure_name = '%s:%s' % (exposure_layer.workspace, exposure_layer.name) # Check metadata assert_bounding_box_matches(exposure_layer, exposure_filename) exp_bbox_string = get_bounding_box_string(exposure_filename) check_layer(exposure_layer, full=True) # Upload hazard data filename = 'eq_yogya_2006.asc' hazard_filename = '%s/%s' % (TESTDATA, filename) hazard_layer = save_to_geonode(hazard_filename, user=self.user, overwrite=True) hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name) # Check metadata assert_bounding_box_matches(hazard_layer, hazard_filename) haz_bbox_string = get_bounding_box_string(hazard_filename) check_layer(hazard_layer, full=True) # Run calculation c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard=hazard_name, exposure_server=INTERNAL_SERVER_URL, exposure=exposure_name, bbox=haz_bbox_string, impact_function='EarthquakeFatalityFunction', keywords='test,fatalities,population,usgs')) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) if 'errors' in data: errors = data['errors'] if errors is not None: msg = ('The server returned the error message: %s' % str(errors)) raise Exception(msg) assert 'success' in data assert 'hazard_layer' in data assert 'exposure_layer' in data assert 'run_duration' in data assert 'run_date' in data assert 'layer' in data assert data['success'] # Download result and check layer_name = data['layer'].split('/')[-1] result_layer = download(INTERNAL_SERVER_URL, layer_name, get_bounding_box_string(hazard_filename)) assert os.path.exists(result_layer.filename) # Check calculated values keywords = result_layer.get_keywords() assert 'caption' in keywords
def test_earthquake_exposure_plugin(self): """Population exposure to individual MMI levels can be computed """ # Upload exposure data for this test # FIXME (Ole): While this dataset is ok for testing, # note that is has been resampled without scaling # so numbers are about 25 times too large. # Consider replacing test populations dataset for good measures, # just in case any one accidentally started using this dataset # for real. name = 'Population_2010' exposure_filename = '%s/%s.asc' % (TESTDATA, name) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) exposure_name = '%s:%s' % (exposure_layer.workspace, exposure_layer.name) # Check metadata assert_bounding_box_matches(exposure_layer, exposure_filename) exp_bbox_string = get_bounding_box_string(exposure_filename) check_layer(exposure_layer, full=True) # Upload hazard data filename = 'Lembang_Earthquake_Scenario.asc' hazard_filename = '%s/%s' % (TESTDATA, filename) hazard_layer = save_to_geonode(hazard_filename, user=self.user, overwrite=True) hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name) # Check metadata assert_bounding_box_matches(hazard_layer, hazard_filename) haz_bbox_string = get_bounding_box_string(hazard_filename) check_layer(hazard_layer, full=True) # Run calculation c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard=hazard_name, exposure_server=INTERNAL_SERVER_URL, exposure=exposure_name, bbox=haz_bbox_string, impact_function='EarthquakePopulationExposureFunction', keywords='test,population,exposure,usgs')) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) if 'errors' in data: errors = data['errors'] if errors is not None: msg = ('The server returned the error message: %s' % str(errors)) raise Exception(msg) assert 'success' in data assert 'hazard_layer' in data assert 'exposure_layer' in data assert 'run_duration' in data assert 'run_date' in data assert 'layer' in data assert data['success'] # Download result and check layer_name = data['layer'].split('/')[-1] result_layer = download(INTERNAL_SERVER_URL, layer_name, get_bounding_box_string(hazard_filename)) assert os.path.exists(result_layer.filename) # Check calculated values keywords = result_layer.get_keywords() assert 'mmi-classes' in keywords assert 'affected-population' in keywords mmi_classes = [int(x) for x in keywords['mmi-classes'].split('_')] count = [float(x) for x in keywords['affected-population'].split('_')] # Brute force count for each population level population = download(INTERNAL_SERVER_URL, exposure_name, get_bounding_box_string(hazard_filename)) intensity = download(INTERNAL_SERVER_URL, hazard_name, get_bounding_box_string(hazard_filename)) # Extract data H = intensity.get_data(nan=0) P = population.get_data(nan=0) brutecount = {} for mmi in mmi_classes: brutecount[mmi] = 0 for i in range(P.shape[0]): for j in range(P.shape[1]): mmi = H[i, j] if not numpy.isnan(mmi): mmi_class = int(round(mmi)) pop = P[i, j] if not numpy.isnan(pop): brutecount[mmi_class] += pop for i, mmi in enumerate(mmi_classes): assert numpy.allclose(count[i], brutecount[mmi], rtol=1.0e-6)
def test_jakarta_flood_study(self): """HKV Jakarta flood study calculated correctly using the API """ # FIXME (Ole): Redo with population as shapefile later # Expected values from HKV expected_values = [2485442, 1537920] # Name files for hazard level, exposure and expected fatalities population = 'Population_Jakarta_geographic' plugin_name = 'FloodImpactFunction' # Upload exposure data for this test exposure_filename = '%s/%s.asc' % (TESTDATA, population) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) workspace = exposure_layer.workspace msg = 'Expected workspace to be "geonode". Got %s' % workspace assert workspace == 'geonode' layer_name = exposure_layer.name msg = 'Expected layer name to be "%s". Got %s' % (population, layer_name) assert layer_name.lower() == population.lower(), msg exposure_name = '%s:%s' % (workspace, layer_name) # Check metadata assert_bounding_box_matches(exposure_layer, exposure_filename) exp_bbox_string = get_bounding_box_string(exposure_filename) check_layer(exposure_layer, full=True) # Now we know that exposure layer is good, lets upload some # hazard layers and do the calculations i = 0 for filename in ['Flood_Current_Depth_Jakarta_geographic.asc', 'Flood_Design_Depth_Jakarta_geographic.asc']: hazard_filename = os.path.join(TESTDATA, filename) exposure_filename = os.path.join(TESTDATA, population) # Save hazard_filename = '%s/%s' % (TESTDATA, filename) hazard_layer = save_to_geonode(hazard_filename, user=self.user, overwrite=True) hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name) # Check metadata assert_bounding_box_matches(hazard_layer, hazard_filename) haz_bbox_string = get_bounding_box_string(hazard_filename) check_layer(hazard_layer, full=True) # Run calculation c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard=hazard_name, exposure_server=INTERNAL_SERVER_URL, exposure=exposure_name, bbox=exp_bbox_string, impact_function=plugin_name, keywords='test,flood,HKV')) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) if 'errors' in data: errors = data['errors'] if errors is not None: raise Exception(errors) assert 'hazard_layer' in data assert 'exposure_layer' in data assert 'run_duration' in data assert 'run_date' in data assert 'layer' in data # Do calculation manually and check result hazard_raster = read_layer(hazard_filename) H = hazard_raster.get_data(nan=0) exposure_raster = read_layer(exposure_filename + '.asc') P = exposure_raster.get_data(nan=0) # Calculate impact manually pixel_area = 2500 I = numpy.where(H > 0.1, P, 0) / 100000.0 * pixel_area # Verify correctness against results from HKV res = sum(I.flat) ref = expected_values[i] #print filename, 'Result=%f' % res, ' Expected=%f' % ref #print 'Pct relative error=%f' % (abs(res-ref)*100./ref) msg = 'Got result %f but expected %f' % (res, ref) assert numpy.allclose(res, ref, rtol=1.0e-2), msg # Verify correctness of result # Download result and check layer_name = data['layer'].split('/')[-1] result_layer = download(INTERNAL_SERVER_URL, layer_name, get_bounding_box_string(hazard_filename)) assert os.path.exists(result_layer.filename) calculated_raster = read_layer(result_layer.filename) C = calculated_raster.get_data(nan=0) # FIXME (Ole): Bring this back # Check caption #caption = calculated_raster.get_caption() #print #print caption #expct = 'people' #msg = ('Caption %s did not contain expected ' # 'keyword %s' % (caption, expct)) #assert expct in caption, msg # Compare shape and extrema msg = ('Shape of calculated raster differs from reference raster: ' 'C=%s, I=%s' % (C.shape, I.shape)) assert numpy.allclose(C.shape, I.shape, rtol=1e-12, atol=1e-12), msg msg = ('Minimum of calculated raster differs from reference ' 'raster: ' 'C=%s, I=%s' % (numpy.nanmin(C), numpy.nanmin(I))) assert numpy.allclose(numpy.nanmin(C), numpy.nanmin(I), rtol=1e-6, atol=1e-12), msg msg = ('Maximum of calculated raster differs from reference ' 'raster: ' 'C=%s, I=%s' % (numpy.nanmax(C), numpy.nanmax(I))) assert numpy.allclose(numpy.nanmax(C), numpy.nanmax(I), rtol=1e-6, atol=1e-12), msg # Compare every single value numerically (a bit loose - # probably due to single precision conversions when # data flows through geonode) # # FIXME: Not working - but since this test is about # issue #162 we'll leave it for now. TODO with NAN # Manually verified that the two expected values are correct, # though. #msg = 'Array values of written raster array were not as expected' #print C #print I #print numpy.amax(numpy.abs(C-I)) #assert numpy.allclose(C, I, rtol=1e-2, atol=1e-5), msg # Check that extrema are in range xmin, xmax = calculated_raster.get_extrema() assert numpy.alltrue(C[-numpy.isnan(C)] >= xmin), msg assert numpy.alltrue(C[-numpy.isnan(C)] <= xmax) assert numpy.alltrue(C[-numpy.isnan(C)] >= 0) i += 1
def test_the_earthquake_fatality_estimation_allen(self): """Fatality computation computed correctly with GeoServer Data """ # Simulate bounding box from application viewport_bbox_string = '104.3,-8.2,110.04,-5.17' # Upload exposure data for this test name = 'Population_2010' exposure_filename = '%s/%s.asc' % (TESTDATA, name) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) workspace = exposure_layer.workspace msg = 'Expected workspace to be "geonode". Got %s' % workspace assert workspace == 'geonode' layer_name = exposure_layer.name msg = 'Expected layer name to be "%s". Got %s' % (name, layer_name) assert layer_name == name.lower(), msg exposure_name = '%s:%s' % (workspace, layer_name) # Check metadata assert_bounding_box_matches(exposure_layer, exposure_filename) exp_bbox_string = get_bounding_box_string(exposure_filename) check_layer(exposure_layer, full=True) # Now we know that exposure layer is good, lets upload some # hazard layers and do the calculations filename = 'Lembang_Earthquake_Scenario.asc' # Save hazard_filename = '%s/%s' % (TESTDATA, filename) hazard_layer = save_to_geonode(hazard_filename, user=self.user, overwrite=True) hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name) # Check metadata assert_bounding_box_matches(hazard_layer, hazard_filename) haz_bbox_string = get_bounding_box_string(hazard_filename) check_layer(hazard_layer, full=True) # Run calculation c = Client() rv = c.post('/impact/api/calculate/', data=dict( hazard_server=INTERNAL_SERVER_URL, hazard=hazard_name, exposure_server=INTERNAL_SERVER_URL, exposure=exposure_name, #bbox=viewport_bbox_string, bbox=exp_bbox_string, # This one reproduced the # crash for lembang impact_function='EarthquakeFatalityFunction', keywords='test,shakemap,usgs')) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') data = json.loads(rv.content) if 'errors' in data: errors = data['errors'] if errors is not None: msg = ('The server returned the error message: %s' % str(errors)) raise Exception(msg) assert 'success' in data assert 'hazard_layer' in data assert 'exposure_layer' in data assert 'run_duration' in data assert 'run_date' in data assert 'layer' in data assert data['success'] # Download result and check layer_name = data['layer'].split('/')[-1] result_layer = download(INTERNAL_SERVER_URL, layer_name, get_bounding_box_string(hazard_filename)) assert os.path.exists(result_layer.filename)
def test_metadata(self): """Metadata is retrieved correctly for both raster and vector data """ # Upload test data filenames = [ 'Lembang_Earthquake_Scenario.asc', 'Earthquake_Ground_Shaking.asc', 'lembang_schools.shp', 'Padang_WGS84.shp' ] layers = [] paths = [] for filename in filenames: basename, ext = os.path.splitext(filename) path = os.path.join(TESTDATA, filename) layer = save_to_geonode(path, user=self.user, overwrite=True) # Record layer and file layers.append(layer) paths.append(path) # Check integrity for i, layer in enumerate(layers): if filenames[i].endswith('.shp'): layer_type = 'vector' elif filenames[i].endswith('.asc'): layer_type = 'raster' else: msg = ('Unknown layer extension in %s. ' 'Expected .shp or .asc' % filenames[i]) raise Exception(msg) layer_name = '%s:%s' % (layer.workspace, layer.name) metadata = get_metadata(INTERNAL_SERVER_URL, layer_name) assert 'id' in metadata assert 'title' in metadata assert 'layer_type' in metadata assert 'keywords' in metadata assert 'bounding_box' in metadata assert len(metadata['bounding_box']) == 4 # Check integrity between Django layer and file assert_bounding_box_matches(layer, paths[i]) # Check integrity between file and OWS metadata ref_bbox = get_bounding_box(paths[i]) msg = ('Bounding box from OWS did not match bounding box ' 'from file. They are\n' 'From file %s: %s\n' 'From OWS: %s' % (paths[i], ref_bbox, metadata['bounding_box'])) assert numpy.allclose(metadata['bounding_box'], ref_bbox), msg assert layer.name == metadata['title'] assert layer_name == metadata['id'] assert layer_type == metadata['layer_type'] # Check keywords if layer_type == 'raster': category = 'hazard' subcategory = 'earthquake' elif layer_type == 'vector': category = 'exposure' subcategory = 'building' else: msg = 'Unknown layer type %s' % layer_type raise Exception(msg) keywords = metadata['keywords'] msg = 'Did not find key "category" in keywords: %s' % keywords assert 'category' in keywords, msg msg = 'Did not find key "subcategory" in keywords: %s' % keywords assert 'subcategory' in keywords, msg msg = ('Category keyword %s did not match expected %s' % (keywords['category'], category)) assert category == keywords['category'], msg msg = ('Subcategory keyword %s did not match expected %s' % (keywords['subcategory'], category)) assert subcategory == keywords['subcategory'], msg
def test_metadata_twice(self): """Layer metadata can be correctly uploaded multiple times """ # This test reproduces ticket #99 by creating new data, # uploading twice and verifying metadata # Base test data filenames = ['Lembang_Earthquake_Scenario.asc', 'lembang_schools.shp'] for org_filename in filenames: org_basename, ext = os.path.splitext( os.path.join(TESTDATA, org_filename)) # Copy data to temporary unique name basename = unique_filename(dir='/tmp') cmd = '/bin/cp %s.keywords %s.keywords' % (org_basename, basename) os.system(cmd) cmd = '/bin/cp %s.prj %s.prj' % (org_basename, basename) os.system(cmd) if ext == '.asc': layer_type = 'raster' filename = '%s.asc' % basename cmd = '/bin/cp %s.asc %s' % (org_basename, filename) os.system(cmd) elif ext == '.shp': layer_type = 'vector' filename = '%s.shp' % basename for e in ['shp', 'shx', 'sbx', 'sbn', 'dbf']: cmd = '/bin/cp %s.%s %s.%s' % (org_basename, e, basename, e) os.system(cmd) else: msg = ('Unknown layer extension in %s. ' 'Expected .shp or .asc' % filename) raise Exception(msg) # Repeat multiple times for i in range(3): # Upload layer = save_to_geonode(filename, user=self.user, overwrite=True) # Get metadata layer_name = '%s:%s' % (layer.workspace, layer.name) metadata = get_metadata(INTERNAL_SERVER_URL, layer_name) # Verify assert 'id' in metadata assert 'title' in metadata assert 'layer_type' in metadata assert 'keywords' in metadata assert 'bounding_box' in metadata assert len(metadata['bounding_box']) == 4 # Check integrity between Django layer and file assert_bounding_box_matches(layer, filename) # Check integrity between file and OWS metadata ref_bbox = get_bounding_box(filename) msg = ('Bounding box from OWS did not match bounding box ' 'from file. They are\n' 'From file %s: %s\n' 'From OWS: %s' % (filename, ref_bbox, metadata['bounding_box'])) assert numpy.allclose(metadata['bounding_box'], ref_bbox), msg assert layer.name == metadata['title'] assert layer_name == metadata['id'] assert layer_type == metadata['layer_type'] # Check keywords if layer_type == 'raster': category = 'hazard' subcategory = 'earthquake' elif layer_type == 'vector': category = 'exposure' subcategory = 'building' else: msg = 'Unknown layer type %s' % layer_type raise Exception(msg) keywords = metadata['keywords'] msg = 'Did not find key "category" in keywords: %s' % keywords assert 'category' in keywords, msg msg = ('Did not find key "subcategory" in keywords: %s' % keywords) assert 'subcategory' in keywords, msg msg = ('Category keyword %s did not match expected %s' % (keywords['category'], category)) assert category == keywords['category'], msg msg = ('Subcategory keyword %s did not match expected %s' % (keywords['subcategory'], category)) assert subcategory == keywords['subcategory'], msg