def test_get_meta(self, open_dataset_mock, isfile): dataset_path = "/path/to/dataset" isfile.return_value = True self.task_process.return_value = Mock(exitcode=0) mock_open_dataset = Mock(spec=gdal.Dataset) mock_open_dataset.RasterCount = 0 open_dataset_mock.return_value = mock_open_dataset mock_open_dataset.GetDriver.return_value.ShortName = "gtiff" expected_meta = {"driver": "gtiff", "is_raster": True, "nodata": None} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta) mock_open_dataset.RasterCount = 2 mock_open_dataset.GetRasterBand.return_value.GetNoDataValue.return_value = -32768.0 expected_meta = { "driver": "gtiff", "is_raster": True, "nodata": -32768.0 } returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta) mock_open_dataset = Mock(spec=ogr.DataSource) open_dataset_mock.return_value = mock_open_dataset mock_open_dataset.GetDriver.return_value.GetName.return_value = "gpkg" expected_meta = {"driver": "gpkg", "is_raster": False, "nodata": None} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta) open_dataset_mock.return_value = None expected_meta = {"driver": None, "is_raster": None, "nodata": None} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta)
def test_get_meta(self, open_ds_mock, isfile): dataset_path = "/path/to/dataset" isfile.return_value = True self.task_process.return_value = Mock(exitcode=0) ds_mock = Mock(spec=gdal.Dataset) ds_mock.RasterCount = 0 open_ds_mock.return_value = ds_mock ds_mock.GetDriver.return_value.ShortName = 'gtiff' expected_meta = {'driver': 'gtiff', 'is_raster': True, 'nodata': None} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta) ds_mock.RasterCount = 2 ds_mock.GetRasterBand.return_value.GetNoDataValue.return_value = -32768.0 expected_meta = { 'driver': 'gtiff', 'is_raster': True, 'nodata': -32768.0 } returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta) ds_mock = Mock(spec=ogr.DataSource) open_ds_mock.return_value = ds_mock ds_mock.GetDriver.return_value.GetName.return_value = 'gpkg' expected_meta = {'driver': 'gpkg', 'is_raster': False, 'nodata': None} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta) open_ds_mock.return_value = None expected_meta = {'driver': None, 'is_raster': None, 'nodata': None} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta)
def test_get_meta(self, open_ds_mock, isfile): dataset_path = "/path/to/dataset" isfile.return_value = True self.task_process.return_value = Mock(exitcode=0) ds_mock = Mock(spec=gdal.Dataset) ds_mock.RasterCount = 0 open_ds_mock.return_value = ds_mock ds_mock.GetDriver.return_value.ShortName = 'gtiff' expected_meta = {'driver': 'gtiff', 'is_raster': True, 'nodata': None} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta) ds_mock.RasterCount = 2 ds_mock.GetRasterBand.return_value.GetNoDataValue.return_value = -32768.0 expected_meta = {'driver': 'gtiff', 'is_raster': True, 'nodata': -32768.0} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta) ds_mock = Mock(spec=ogr.DataSource) open_ds_mock.return_value = ds_mock ds_mock.GetDriver.return_value.GetName.return_value = 'gpkg' expected_meta = {'driver': 'gpkg', 'is_raster': False, 'nodata': None} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta) open_ds_mock.return_value = None expected_meta = {'driver': None, 'is_raster': None, 'nodata': None} returned_meta = get_meta(dataset_path) self.assertEqual(expected_meta, returned_meta)
def file_to_geojson(in_path: str): """ :param in_path: A str path to a file. :return: A geojson object if available """ dir_name = os.path.dirname(in_path) file_path = pathlib.Path(in_path) try: out_path = os.path.join(dir_name, "out_{0}.geojson".format(file_path.stem)) if file_path.suffix == ".zip": if unzip_file(in_path, dir_name): has_shp = False for unzipped_file in os.listdir(dir_name): if unzipped_file.endswith(".shp"): in_path = os.path.join(dir_name, unzipped_file) has_shp = True break if not has_shp: raise Exception("Zip file does not contain a shp") meta = get_meta(in_path, is_raster=False) if not meta["driver"] or meta["is_raster"]: out_path = polygonize(in_path, out_path) else: out_path = convert_vector(in_path, out_path, driver="geojson") if os.path.exists(out_path): geojson = read_json_file(out_path) return geojson raise Exception("An unknown error occurred while processing the file") except Exception as e: logger.error(e, exc_info=True) raise e finally: if os.path.exists(dir_name): shutil.rmtree(dir_name)
def file_to_geojson(in_memory_file): """ :param in_memory_file: A WSGI In memory file :return: A geojson object if available """ stage_dir = settings.EXPORT_STAGING_ROOT.rstrip('\/') uid = str(uuid4()) dir = os.path.join(stage_dir, uid) try: os.mkdir(dir) file_name = in_memory_file.name file_name, file_extension = os.path.splitext(file_name) if not file_name or not file_extension: raise Exception('No file type detected') # Remove all non-word characters file_name = re.sub(r"[^\w\s]", '', file_name) # Replace all whitespace with a single underscore file_name = re.sub(r"\s+", '_', file_name).lower() in_path = os.path.join(dir, 'in_{0}{1}'.format(file_name, file_extension)) out_path = os.path.join(dir, 'out_{0}.geojson'.format(file_name)) write_uploaded_file(in_memory_file, in_path) if file_extension == '.zip': if unzip_file(in_path, dir): has_shp = False for unzipped_file in os.listdir(dir): if unzipped_file.endswith('.shp'): in_path = os.path.join(dir, unzipped_file) has_shp = True break if not has_shp: raise Exception('Zip file does not contain a shp') meta = get_meta(in_path) if not meta['driver'] or meta['is_raster']: raise Exception("Could not find the proper driver to handle this file") cmd_template = Template("ogr2ogr -f $fmt $out_ds $in_ds") cmd = cmd_template.safe_substitute({ 'fmt': 'geojson', 'out_ds': out_path, 'in_ds': in_path }) try: proc = subprocess.Popen(cmd, shell=True, executable='/bin/bash') proc.wait() except Exception as e: logger.debug(e) raise Exception('Failed to convert file') if os.path.exists(out_path): geojson = read_json_file(out_path) return geojson raise Exception('An unknown error occurred while processing the file') except Exception as e: logger.error(e) raise e finally: if os.path.exists(dir): shutil.rmtree(dir)
def get_coverage_with_requests(self): logger.info("Using admin configuration for the WCS request.") service = self.config.get("service") params = self.config.get("params") if not service: raise Exception( "A service key needs to be defined to include the scale of source in meters" ) coverages = service.get("coverages", params.get("COVERAGE")) coverages = str(coverages).split(",") if not coverages: logger.error( "No coverages were specified for this provider, " "please specify `coverages` under service or `COVERAGE` under params." # NOQA ) raise Exception("Data source incorrectly configured.") scale = float(service.get("scale")) params["service"] = "WCS" width, height = gdalutils.get_dimensions(self.bbox, scale) tile_bboxes = gdalutils.get_chunked_bbox(self.bbox, (width, height)) geotiffs = [] session = get_or_update_session(slug=self.slug, **self.config) for idx, coverage in enumerate(coverages): params["COVERAGE"] = coverage file_path, ext = os.path.splitext(self.out) try: for ( _bbox_idx, _tile_bbox, ) in enumerate(tile_bboxes): outfile = "{0}-{1}-{2}{3}".format(file_path, idx, _bbox_idx, ext) try: os.remove(outfile) except OSError: pass # Setting this to arbitrarily high values improves the computed # resolution but makes the requests slow down. # If it is set in the config, use that value, otherwise compute approximate res based on scale if self.config.get("tile_size", None) is None: tile_x, tile_y = gdalutils.get_dimensions( _tile_bbox, scale) params["width"] = tile_x params["height"] = tile_y else: params["width"] = self.config.get("tile_size") params["height"] = self.config.get("tile_size") params["bbox"] = ",".join(map(str, _tile_bbox)) req = session.get(self.service_url, params=params, stream=True) try: size = int(req.headers.get("content-length")) except (ValueError, TypeError): if req.content: size = len(req.content) else: raise Exception( "Overpass Query failed to return any data") if not req: logger.error(req.content) raise Exception("WCS request for {0} failed.".format( self.name)) CHUNK = 1024 * 1024 * 2 # 2MB chunks from audit_logging.file_logging import logging_open with logging_open(outfile, "wb", user_details=self.user_details) as fd: for chunk in req.iter_content(CHUNK): fd.write(chunk) size += CHUNK geotiffs += [outfile] except Exception as e: logger.error(e) raise Exception("There was an error writing the file to disk.") if len(geotiffs) > 1: self.out = gdalutils.merge_geotiffs(geotiffs, self.out, task_uid=self.task_uid) else: shutil.copy(geotiffs[0], self.out) if not os.path.isfile(self.out): raise Exception("Nothing was returned from the WCS service.") if not gdalutils.get_meta(self.out).get("is_raster"): with open(self.out, "r") as output_file: logger.error("Content of failed WCS request") logger.error(output_file.read()) raise Exception("The service failed to return a proper response")