def test_get_dimensions(self, mock_get_distance): bbox = [0, 1, 2, 3] scale = 10 expected_dim = (10, 20) mock_get_distance.side_effect = [100, 200] dim = get_dimensions(bbox, scale) mock_get_distance.assert_has_calls([ call([bbox[0], bbox[1]], [bbox[2], bbox[1]]), call([bbox[0], bbox[1]], [bbox[0], bbox[3]]) ]) self.assertEqual(dim, expected_dim) expected_dim = (1, 1) mock_get_distance.side_effect = [6, 8] dim = get_dimensions(bbox, scale) self.assertEqual(dim, expected_dim) expected_dim = (1, 5) mock_get_distance.side_effect = [9, 50] dim = get_dimensions(bbox, scale) self.assertEqual(dim, expected_dim) expected_dim = (6, 1) mock_get_distance.side_effect = [60, 8] dim = get_dimensions(bbox, scale) self.assertEqual(dim, expected_dim)
def get_coverage_with_requests(self): logger.info("Using admin configuration for the WCS request.") service = self.config.get('service') params = self.config.get('params') if not service: raise Exception('A service key needs to be defined to include the scale of source in meters') coverages = service.get('coverages', params.get('COVERAGE')) coverages = str(coverages).split(',') if not coverages: logger.error('No coverages were specified for this provider, please specify `coverages` under service or `COVERAGE` under params.') raise Exception("Data source incorrectly configured.") logger.info("Getting Dimensions...") width, height = get_dimensions(self.bbox, float(service.get('scale'))) logger.info("{}, {}".format(width, height)) params['width'] = str(width) params['height'] = str(height) params['service'] = 'WCS' params['bbox'] = ','.join(map(str, self.bbox)) geotiffs = [] for idx, coverage in enumerate(coverages): params['COVERAGE'] = coverage file_path, ext = os.path.splitext(self.out) outfile = '{0}-{1}{2}'.format(file_path, idx, ext) try: os.remove(outfile) except OSError: pass try: req = auth_requests.get(self.service_url, params=params, slug=self.slug, stream=True, verify=getattr(settings, 'SSL_VERIFICATION', True)) logger.info("Getting the coverage: {0}".format(req.url)) try: size = int(req.headers.get('content-length')) except (ValueError, TypeError): if req.content: size = len(req.content) else: raise Exception("Overpass Query failed to return any data") if not req: logger.error(req.content) raise Exception("WCS request for {0} failed.".format(self.name)) CHUNK = 1024 * 1024 * 2 # 2MB chunks from audit_logging.file_logging import logging_open with logging_open(outfile, 'wb', user_details=self.user_details) as fd: for chunk in req.iter_content(CHUNK): fd.write(chunk) size += CHUNK geotiffs += [outfile] except Exception as e: logger.error(e) raise Exception("There was an error writing the file to disk.") if len(geotiffs) > 1: self.out = merge_geotiffs(geotiffs, self.out, task_uid=self.task_uid) else: shutil.copy(geotiffs[0], self.out) if not os.path.isfile(self.out): raise Exception("Nothing was returned from the WCS service.")
def test_get_dimensions(self, mock_get_distance): bbox = [0, 1, 2, 3] scale = 10 expected_dim = [10, 20] mock_get_distance.side_effect = [100, 200] dim = get_dimensions(bbox, scale) mock_get_distance.assert_has_calls( [call([bbox[0], bbox[1]], [bbox[2], bbox[1]]), call([bbox[0], bbox[1]], [bbox[0], bbox[3]])]) self.assertEqual(dim, expected_dim)
def get_coverage_with_requests(self): logger.info("Using admin configuration for the WCS request.") service = self.config.get('service') params = self.config.get('params') if not service: raise Exception( 'A service key needs to be defined to include the scale of source in meters' ) coverages = service.get('coverages', params.get('COVERAGE')) coverages = str(coverages).split(',') if not coverages: logger.error( 'No coverages were specified for this provider, please specify `coverages` under service or `COVERAGE` under params.' ) raise Exception("Data source incorrectly configured.") logger.info("Getting Dimensions...") width, height = get_dimensions(self.bbox, float(service.get('scale'))) logger.info("{}, {}".format(width, height)) params['width'] = str(width) params['height'] = str(height) params['service'] = 'WCS' params['bbox'] = ','.join(map(str, self.bbox)) geotiffs = [] for idx, coverage in enumerate(coverages): params['COVERAGE'] = coverage file_path, ext = os.path.splitext(self.out) outfile = '{0}-{1}{2}'.format(file_path, idx, ext) try: os.remove(outfile) except OSError: pass try: req = auth_requests.get(self.service_url, params=params, slug=self.slug, stream=True, verify=getattr(settings, 'SSL_VERIFICATION', True)) logger.info("Getting the coverage: {0}".format(req.url)) try: size = int(req.headers.get('content-length')) except (ValueError, TypeError): if req.content: size = len(req.content) else: raise Exception( "Overpass Query failed to return any data") if not req: logger.error(req.content) raise Exception("WCS request for {0} failed.".format( self.name)) CHUNK = 1024 * 1024 * 2 # 2MB chunks from audit_logging.file_logging import logging_open with logging_open(outfile, 'wb', user_details=self.user_details) as fd: for chunk in req.iter_content(CHUNK): fd.write(chunk) size += CHUNK geotiffs += [outfile] except Exception as e: logger.error(e) raise Exception("There was an error writing the file to disk.") if len(geotiffs) > 1: self.out = merge_geotiffs(geotiffs, self.out, task_uid=self.task_uid) else: shutil.copy(geotiffs[0], self.out) if not os.path.isfile(self.out): raise Exception("Nothing was returned from the WCS service.")
def get_coverage_with_requests(self): logger.info("Using admin configuration for the WCS request.") service = self.config.get("service") params = self.config.get("params") if not service: raise Exception( "A service key needs to be defined to include the scale of source in meters" ) coverages = service.get("coverages", params.get("COVERAGE")) coverages = str(coverages).split(",") if not coverages: logger.error( "No coverages were specified for this provider, " "please specify `coverages` under service or `COVERAGE` under params." # NOQA ) raise Exception("Data source incorrectly configured.") scale = float(service.get("scale")) params["service"] = "WCS" width, height = gdalutils.get_dimensions(self.bbox, scale) tile_bboxes = gdalutils.get_chunked_bbox(self.bbox, (width, height)) geotiffs = [] session = get_or_update_session(slug=self.slug, **self.config) for idx, coverage in enumerate(coverages): params["COVERAGE"] = coverage file_path, ext = os.path.splitext(self.out) try: for ( _bbox_idx, _tile_bbox, ) in enumerate(tile_bboxes): outfile = "{0}-{1}-{2}{3}".format(file_path, idx, _bbox_idx, ext) try: os.remove(outfile) except OSError: pass # Setting this to arbitrarily high values improves the computed # resolution but makes the requests slow down. # If it is set in the config, use that value, otherwise compute approximate res based on scale if self.config.get("tile_size", None) is None: tile_x, tile_y = gdalutils.get_dimensions( _tile_bbox, scale) params["width"] = tile_x params["height"] = tile_y else: params["width"] = self.config.get("tile_size") params["height"] = self.config.get("tile_size") params["bbox"] = ",".join(map(str, _tile_bbox)) req = session.get(self.service_url, params=params, stream=True) try: size = int(req.headers.get("content-length")) except (ValueError, TypeError): if req.content: size = len(req.content) else: raise Exception( "Overpass Query failed to return any data") if not req: logger.error(req.content) raise Exception("WCS request for {0} failed.".format( self.name)) CHUNK = 1024 * 1024 * 2 # 2MB chunks from audit_logging.file_logging import logging_open with logging_open(outfile, "wb", user_details=self.user_details) as fd: for chunk in req.iter_content(CHUNK): fd.write(chunk) size += CHUNK geotiffs += [outfile] except Exception as e: logger.error(e) raise Exception("There was an error writing the file to disk.") if len(geotiffs) > 1: self.out = gdalutils.merge_geotiffs(geotiffs, self.out, task_uid=self.task_uid) else: shutil.copy(geotiffs[0], self.out) if not os.path.isfile(self.out): raise Exception("Nothing was returned from the WCS service.") if not gdalutils.get_meta(self.out).get("is_raster"): with open(self.out, "r") as output_file: logger.error("Content of failed WCS request") logger.error(output_file.read()) raise Exception("The service failed to return a proper response")