def get_bounding_boxes(haz_metadata, exp_metadata, req_bbox): """Check and get appropriate bounding boxes for input layers Input haz_metadata: Metadata for hazard layer exp_metadata: Metadata for exposure layer req_bbox: Bounding box (string) as requested by HTML POST. Output haz_bbox: Bounding box to be used for hazard layer. exp_bbox: Bounding box to be used for exposure layer imp_bbox: Bounding box to be used for resulting impact layer Note exp_bbox and imp_bbox are the same and calculated as the intersection among hazard, exposure and viewport bounds. haz_bbox may be grown by one pixel size in case exposure data is vector data to make sure points always can be interpolated """ # Input checks msg = ('Invalid bounding box %s (%s). ' 'It must be a string' % (str(req_bbox), type(req_bbox))) assert isinstance(req_bbox, basestring), msg check_bbox_string(req_bbox) # Get bounding boxes for layers and viewport haz_bbox = haz_metadata['bounding_box'] exp_bbox = exp_metadata['bounding_box'] vpt_bbox = bboxstring2list(req_bbox) # New bounding box for data common to hazard, exposure and viewport # Download only data within this intersection intersection_bbox = bbox_intersection(vpt_bbox, haz_bbox, exp_bbox) if intersection_bbox is None: # Bounding boxes did not overlap msg = ('Bounding boxes of hazard data [%s], exposure data [%s] ' 'and viewport [%s] did not overlap, so no computation was ' 'done. Please make sure you pan to where the data is and ' 'that hazard and exposure data overlaps.' % (bboxlist2string(haz_bbox, decimals=3), bboxlist2string(exp_bbox, decimals=3), bboxlist2string(vpt_bbox, decimals=3))) logger.info(msg) raise Exception(msg) # Grow hazard bbox to buffer this common bbox in case where # hazard is raster and exposure is vector if (haz_metadata['layer_type'] == 'raster' and exp_metadata['layer_type'] == 'vector'): haz_res = haz_metadata['resolution'] haz_bbox = buffered_bounding_box(intersection_bbox, haz_res) else: haz_bbox = intersection_bbox # Usually the intersection bbox is used for both exposure layer and result exp_bbox = imp_bbox = intersection_bbox return haz_bbox, exp_bbox, imp_bbox
def test_bounding_box_conversions(self): """Bounding boxes can be converted between list and string """ # Good ones for x in [[105, -7, 108, -5], [106.5, -6.5, 107, -6], [94.972335, -11.009721, 141.014, 6.073612333333], [105.3, -8.5, 110.0, -5.5], [105.6, -7.8, 110.5, -5.1]]: bbox_string = bboxlist2string(x) bbox_list = bboxstring2list(bbox_string) assert numpy.allclose(x, bbox_list, rtol=1.0e-6, atol=1.0e-6) for x in ['105,-7,108,-5', '106.5, -6.5, 107,-6', '94.972335,-11.009721,141.014,6.073612333333']: bbox_list = bboxstring2list(x) # Check that numbers are numerically consistent assert numpy.allclose([float(z) for z in x.split(',')], bbox_list, rtol=1.0e-6, atol=1.0e-6) # Bad ones for bbox in [[105, -7, 'x', -5], [106.5, -6.5, -6], [94.972335, 0, -11.009721, 141.014, 6]]: try: bbox_string = bboxlist2string(bbox) except: pass else: msg = 'Should have raised exception' raise Exception(msg) for x in ['106.5,-6.5,-6', '106.5,-6.5,-6,4,10', '94.972335,x,141.014,6.07']: try: bbox_list = bboxstring2list(x) except: pass else: msg = 'Should have raised exception: %s' % x raise Exception(msg)
def calculate(request, save_output=save_to_geonode): start = datetime.datetime.now() if request.method == 'GET': # FIXME: Add a basic form here to be able to generate the POST request. return HttpResponse('This should be accessed by robots, not humans.' 'In other words using HTTP POST instead of GET.') elif request.method == 'POST': data = request.POST impact_function_name = data['impact_function'] hazard_server = data['hazard_server'] hazard_layer = data['hazard'] exposure_server = data['exposure_server'] exposure_layer = data['exposure'] bbox = data['bbox'] keywords = data['keywords'] if request.user.is_anonymous(): theuser = get_valid_user() else: theuser = request.user # Create entry in database calculation = Calculation(user=theuser, run_date=start, hazard_server=hazard_server, hazard_layer=hazard_layer, exposure_server=exposure_server, exposure_layer=exposure_layer, impact_function=impact_function_name, success=False) try: # Input checks msg = 'This cannot happen :-)' assert isinstance(bbox, basestring), msg check_bbox_string(bbox) # Find the intersection of bounding boxes for viewport, # hazard and exposure. vpt_bbox = bboxstring2list(bbox) haz_bbox = get_metadata(hazard_server, hazard_layer)['bounding_box'] exp_bbox = get_metadata(exposure_server, exposure_layer)['bounding_box'] # Impose minimum bounding box size (as per issue #101). # FIXME (Ole): This will need to be revisited in conjunction with # raster resolutions at some point. min_res = 0.00833334 eps = 1.0e-1 vpt_bbox = minimal_bounding_box(vpt_bbox, min_res, eps=eps) haz_bbox = minimal_bounding_box(haz_bbox, min_res, eps=eps) exp_bbox = minimal_bounding_box(exp_bbox, min_res, eps=eps) # New bounding box for data common to hazard, exposure and viewport # Download only data within this intersection intersection = bbox_intersection(vpt_bbox, haz_bbox, exp_bbox) if intersection is None: # Bounding boxes did not overlap msg = ('Bounding boxes of hazard data, exposure data and ' 'viewport did not overlap, so no computation was ' 'done. Please try again.') logger.info(msg) raise Exception(msg) bbox = bboxlist2string(intersection) plugin_list = get_plugins(impact_function_name) _, impact_function = plugin_list[0].items()[0] impact_function_source = inspect.getsource(impact_function) calculation.impact_function_source = impact_function_source calculation.bbox = bbox calculation.save() msg = 'Performing requested calculation' logger.info(msg) # Download selected layer objects msg = ('- Downloading hazard layer %s from %s' % (hazard_layer, hazard_server)) logger.info(msg) H = download(hazard_server, hazard_layer, bbox) msg = ('- Downloading exposure layer %s from %s' % (exposure_layer, exposure_server)) logger.info(msg) E = download(exposure_server, exposure_layer, bbox) # Calculate result using specified impact function msg = ('- Calculating impact using %s' % impact_function) logger.info(msg) impact_filename = calculate_impact(layers=[H, E], impact_fcn=impact_function) # Upload result to internal GeoServer msg = ('- Uploading impact layer %s' % impact_filename) logger.info(msg) result = save_output(impact_filename, title='output_%s' % start.isoformat(), user=theuser) except Exception, e: #FIXME: Reimplement error saving for calculation logger.error(e) errors = e.__str__() trace = exception_format(e) calculation.errors = errors calculation.stacktrace = trace calculation.save() jsondata = json.dumps({'errors': errors, 'stacktrace': trace}) return HttpResponse(jsondata, mimetype='application/json')
def test_specified_raster_resolution(self): """Raster layers can be downloaded with specific resolution This is another test for ticket #103 Native test data: maumere....asc ncols 931 nrows 463 cellsize 0.00018 Population_Jakarta ncols 638 nrows 649 cellsize 0.00045228819716044 Population_2010 ncols 5525 nrows 2050 cellsize 0.0083333333333333 Here we download it at a range of fixed resolutions that are both coarser and finer, and check that the dimensions of the downloaded matrix are as expected. We also check that the extrema of the subsampled matrix are sane """ for test_filename in [ 'maumere_aos_depth_20m_land_wgs84.asc', 'Population_Jakarta_geographic.asc', 'Population_2010.asc' ]: hazard_filename = ('%s/%s' % (TESTDATA, test_filename)) # Get reference values H = read_layer(hazard_filename) depth_min_ref, depth_max_ref = H.get_extrema() native_resolution = H.get_resolution() # Upload to internal geonode hazard_layer = save_to_geonode(hazard_filename, user=self.user) hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name) # Test for a range of resolutions for res in [ 0.02, 0.01, 0.005, 0.002, 0.001, 0.0005, # Coarser 0.0002, 0.0001, 0.00006, 0.00003 ]: # Finer # To save time don't do finest resolution for the # two population sets if test_filename.startswith('Population') and res < 0.00006: break # Set bounding box bbox = get_bounding_box_string(hazard_filename) compare_extrema = True if test_filename == 'Population_2010.asc': # Make bbox small for finer resolutions to # save time and to test that as well. # However, extrema obviously won't match those # of the full dataset. Once we can clip # datasets, we can remove this restriction. if res < 0.005: bbox = '106.685974,-6.373421,106.974534,-6.079886' compare_extrema = False bb = bboxstring2list(bbox) # Download data at specified resolution H = download(INTERNAL_SERVER_URL, hazard_name, bbox, resolution=res) A = H.get_data() # Verify that data has the requested bobx and resolution actual_bbox = H.get_bounding_box() msg = ('Bounding box for %s was not as requested. I got %s ' 'but ' 'expected %s' % (hazard_name, actual_bbox, bb)) assert numpy.allclose(actual_bbox, bb, rtol=1.0e-6) # FIXME (Ole): How do we sensibly resolve the issue with # resx, resy vs one resolution (issue #173) actual_resolution = H.get_resolution()[0] # FIXME (Ole): Resolution is often far from the requested # see issue #102 # Here we have to accept up to 5% tolerance102 = 5.0e-2 msg = ('Resolution of %s was not as requested. I got %s but ' 'expected %s' % (hazard_name, actual_resolution, res)) assert numpy.allclose(actual_resolution, res, rtol=tolerance102), msg # Determine expected shape from bbox (W, S, E, N) ref_rows = int(round((bb[3] - bb[1]) / res)) ref_cols = int(round((bb[2] - bb[0]) / res)) # Compare shapes (generally, this may differ by 1) msg = ('Shape of downloaded raster was [%i, %i]. ' 'Expected [%i, %i].' % (A.shape[0], A.shape[1], ref_rows, ref_cols)) assert (ref_rows == A.shape[0] and ref_cols == A.shape[1]), msg # Assess that the range of the interpolated data is sane if not compare_extrema: continue # For these test sets we get exact match of the minimum msg = ( 'Minimum of %s resampled at resolution %f ' 'was %f. Expected %f.' % (hazard_layer.name, res, numpy.nanmin(A), depth_min_ref)) assert numpy.allclose(depth_min_ref, numpy.nanmin(A), rtol=0.0, atol=0.0), msg # At the maximum it depends on the subsampling msg = ( 'Maximum of %s resampled at resolution %f ' 'was %f. Expected %f.' % (hazard_layer.name, res, numpy.nanmax(A), depth_max_ref)) if res < native_resolution[0]: # When subsampling to finer resolutions we expect a # close match assert numpy.allclose(depth_max_ref, numpy.nanmax(A), rtol=1.0e-10, atol=1.0e-8), msg elif res < native_resolution[0] * 10: # When upsampling to coarser resolutions we expect # ballpark match (~20%) assert numpy.allclose(depth_max_ref, numpy.nanmax(A), rtol=0.17, atol=0.0), msg else: # Upsampling to very coarse resolutions, just want sanity assert 0 < numpy.nanmax(A) <= depth_max_ref