def setUp(self): super(GDALDatasetTestCase, self).setUp() _, self.tmppath = tempfile.mkstemp("." + self.getFileExtension("raster")) data = self.getResponseData() if isinstance(data, binary_type): mode = 'wb' else: mode = 'w' with open(self.tmppath, mode) as f: f.write(data) gdal.AllRegister() exp_path = os.path.join(self.getExpectedFileDir(), self.getExpectedFileName("raster")) try: self.res_ds = gdal.Open(self.tmppath, gdal.GA_ReadOnly) except RuntimeError as e: self.fail("Response could not be opened with GDAL. Error was %s" % e) try: self.exp_ds = gdal.Open(exp_path, gdal.GA_ReadOnly) except RuntimeError: self.skipTest("Expected response in '%s' is not present" % exp_path)
def open_gdal(obj): if isinstance(obj, gdal.Dataset): return obj try: return gdal.Open(obj) except RuntimeError: return None
def connect(self, coverage, data_items, layer, options): data = data_items[0].path if coverage.grid.is_referenceable: vrt_path = join("/vsimem", uuid4().hex) reftools.create_rectified_vrt(data, vrt_path) data = vrt_path layer.setMetaData("eoxs_ref_data", data) if not layer.metadata.get("eoxs_wrap_dateline") == "true": layer.data = data else: sr = coverage.grid.spatial_reference extent = coverage.extent e = wrap_extent_around_dateline(extent, sr.srid) vrt_path = join("/vsimem", uuid4().hex) ds = gdal.Open(data) vrt_ds = create_simple_vrt(ds, vrt_path) size_x = ds.RasterXSize size_y = ds.RasterYSize dx = abs(e[0] - e[2]) / size_x dy = abs(e[1] - e[3]) / size_y vrt_ds.SetGeoTransform([e[0], dx, 0, e[3], 0, -dy]) vrt_ds = None layer.data = vrt_path
def connect(self, coverage, data_items, layer, options): filtered = filter(lambda d: d.semantic.startswith("bands"), data_items) data = connect(filtered[0]) if isinstance(coverage, models.ReferenceableDataset): vrt_path = join("/vsimem", uuid4().hex) reftools.create_rectified_vrt(data, vrt_path) data = vrt_path layer.setMetaData("eoxs_ref_data", data) if not layer.metadata.get("eoxs_wrap_dateline") == "true": layer.data = data else: e = wrap_extent_around_dateline(coverage.extent, coverage.srid) vrt_path = join("/vsimem", uuid4().hex) ds = gdal.Open(data) vrt_ds = create_simple_vrt(ds, vrt_path) size_x = ds.RasterXSize size_y = ds.RasterYSize dx = abs(e[0] - e[2]) / size_x dy = abs(e[1] - e[3]) / size_y vrt_ds.SetGeoTransform([e[0], dx, 0, e[3], 0, -dy]) vrt_ds = None layer.data = vrt_path
def _testBinaryComparison(self, file_type, data=None): """ Helper function for the `testBinaryComparisonRaster` function. """ expected_path = os.path.join(self.getExpectedFileDir(), self.getExpectedFileName(file_type)) response_path = os.path.join(self.getResponseFileDir(), self.getResponseFileName(file_type)) actual_response = None if data is None: if file_type in ("raster", "html"): actual_response = self.getResponseData() elif file_type == "xml": actual_response = self.getXMLData() else: self.fail("Unknown file_type '%s'." % file_type) else: actual_response = data # read the expected response, either binary or as string try: if isinstance(actual_response, binary_type): open_type = 'rb' else: open_type = 'r' with open(expected_path, open_type) as f: expected = f.read() except IOError: expected = None if expected != actual_response: if self.getFileExtension("raster") in ("hdf", "nc"): self.skipTest( "Skipping binary comparison for HDF or NetCDF file '%s'." % expected_path) # save the contents of the file if isinstance(actual_response, binary_type): open_type = 'wb' else: open_type = 'w' with open(response_path, open_type) as f: f.write(actual_response) if file_type == "raster": try: gdal.Open(expected_path) except RuntimeError: self.skipTest("Expected response in '%s' is not present" % expected_path) if expected is None: self.skipTest("Expected response in '%s' is not present" % expected_path) else: self.fail("Response returned in '%s' is not equal to expected " "response in '%s'." % (response_path, expected_path))
def testBinaryComparisonRaster(self): response_path = os.path.join(self.getResponseFileDir(), self.getResponseFileName("raster")) expected_path = os.path.join(self.getExpectedFileDir(), self.getExpectedFileName("raster")) # creates a response image that contains the encoded text of the response xml file doc = etree.fromstring(self.prepareXMLData(self.getXMLData())) try: encodedText = doc.xpath( '//wps:ComplexData', namespaces={'wps': 'http://www.opengis.net/wps/1.0.0'})[0].text except IndexError: self.fail('No complex data found in the XML tree') _, self.tmppath = tempfile.mkstemp("." + self.getFileExtension("raster")) with open(self.tmppath, 'wb') as f: f.write(b64decode(encodedText)) gdal.AllRegister() exp_path = os.path.join(self.getExpectedFileDir(), self.getExpectedFileName("raster")) try: self.res_ds = gdal.Open(self.tmppath, gdal.GA_ReadOnly) except RuntimeError as e: self.fail("Response could not be opened with GDAL. Error was %s" % e) try: self.exp_ds = gdal.Open(expected_path, gdal.GA_ReadOnly) except RuntimeError: self.skipTest("Expected response in '%s' is not present" % exp_path) # compare the size self.assertEqual((self.res_ds.RasterXSize, self.res_ds.RasterYSize), (self.exp_ds.RasterXSize, self.exp_ds.RasterYSize)) # compare the band count self.assertEqual(self.res_ds.RasterCount, self.exp_ds.RasterCount)
def from_file(cls, filename, env=None): env = env or {} ds = gdal.Open(filename) size = (ds.RasterXSize, ds.RasterYSize) extent = gdal.get_extent(ds) mode = _get_ds_mode(ds) return cls( filename, env, filename, size, extent, ds.GetProjection(), mode, None )
def _openDatasets(self): _, self.tmppath = tempfile.mkstemp("." + self.getFileExtension("raster")) f = open(self.tmppath, "w") f.write(self.getResponseData()) f.close() gdal.AllRegister() exp_path = os.path.join(self.getExpectedFileDir(), self.getExpectedFileName("raster")) try: self.res_ds = gdal.Open(self.tmppath, gdal.GA_ReadOnly) except RuntimeError, e: self.fail("Response could not be opened with GDAL. Error was %s" % e)
def __init__(self, fname): # get the dataset and its params gdal.AllRegister() ds = gdal.Open(fname) self.fileName = fname self.driverName = ds.GetDriver().LongName self.size = (ds.RasterYSize, ds.RasterXSize, ds.RasterCount) self.GCPCount = ds.GetGCPCount() self.GCPProjection = ds.GetGCPProjection() self.Projection = ds.GetProjection() self.ProjectionRef = ds.GetProjectionRef() self.GeoTransform = ds.GetGeoTransform() self.GCP = ds.GetGCPs() self.isRectified = bool(self.Projection) self.isReferenceable = bool(self.GCPProjection) and (self.GCPCount > 0)
def _read_metadata_from_data(self, data_item, retrieved_metadata, cache): metadata_component = MetadataComponent(env) ds = gdal.Open(connect(data_item, cache)) reader = metadata_component.get_reader_by_test(ds) if reader: values = reader.read(ds) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): retrieved_metadata.setdefault(key, value) ds = None
def warp_fields(coverages, field_name, bbox, crs, width, height): driver = gdal.GetDriverByName('MEM') out_ds = driver.Create( '', width, height, 1, coverages[0].range_type.get_field(field_name).data_type) out_ds.SetGeoTransform([ bbox[0], (bbox[2] - bbox[0]) / width, 0, bbox[3], 0, -(bbox[3] - bbox[1]) / height, ]) epsg = crss.parseEPSGCode(crs, [crss.fromShortCode]) sr = osr.SpatialReference() sr.ImportFromEPSG(epsg) out_ds.SetProjection(sr.ExportToWkt()) for coverage in coverages: location = coverage.get_location_for_field(field_name) band_index = coverage.get_band_index_for_field(field_name) orig_ds = gdal.open_with_env(location.path, location.env) vrt_filename = None if orig_ds.RasterCount > 1: vrt_filename = '/vsimem/' + uuid4().hex gdal.BuildVRT(vrt_filename, orig_ds, bandList=[band_index]) ds = gdal.Open(vrt_filename) else: ds = orig_ds gdal.Warp(out_ds, ds) ds = None if vrt_filename: gdal.Unlink(vrt_filename) band = out_ds.GetRasterBand(1) return band.ReadAsArray()
def __call__(self, src_ds): logger.info("Applying ColorIndexOptimization") try: dst_ds = create_temp(src_ds.RasterXSize, src_ds.RasterYSize, 1, gdal.GDT_Byte, temp_root=self.temporary_directory) if not self.palette_file: # create a color table as a median of the given dataset ct = gdal.ColorTable() gdal.ComputeMedianCutPCT(src_ds.GetRasterBand(1), src_ds.GetRasterBand(2), src_ds.GetRasterBand(3), 256, ct) else: # copy the color table from the given palette file pct_ds = gdal.Open(self.palette_file) pct_ct = pct_ds.GetRasterBand(1).GetRasterColorTable() if not pct_ct: raise ValueError("The palette file '%s' does not have a " "Color Table." % self.palette_file) ct = pct_ct.Clone() pct_ds = None dst_ds.GetRasterBand(1).SetRasterColorTable(ct) gdal.DitherRGB2PCT(src_ds.GetRasterBand(1), src_ds.GetRasterBand(2), src_ds.GetRasterBand(3), dst_ds.GetRasterBand(1), ct) copy_projection(src_ds, dst_ds) copy_metadata(src_ds, dst_ds) return dst_ds except: cleanup_temp(dst_ds) raise
class GDALDatasetTestCase(RasterTestCase): """ Extended RasterTestCases that open the result with GDAL and perform several tests. """ def tearDown(self): super(GDALDatasetTestCase, self).tearDown() try: del self.res_ds del self.exp_ds os.remove(self.tmppath) except AttributeError: pass def _openDatasets(self): _, self.tmppath = tempfile.mkstemp("." + self.getFileExtension("raster")) f = open(self.tmppath, "w") f.write(self.getResponseData()) f.close() gdal.AllRegister() exp_path = os.path.join(self.getExpectedFileDir(), self.getExpectedFileName("raster")) try: self.res_ds = gdal.Open(self.tmppath, gdal.GA_ReadOnly) except RuntimeError, e: self.fail("Response could not be opened with GDAL. Error was %s" % e) try: self.exp_ds = gdal.Open(exp_path, gdal.GA_ReadOnly) except RuntimeError: self.skipTest("Expected response in '%s' is not present" % exp_path)
def render(self, params): # get coverage related stuff coverage = params.coverage # ReferenceableDataset are not supported in WCS < 2.0 if params.coverage.grid.is_referenceable and params.version: raise NoSuchCoverageException((coverage.identifier, )) data_locations = self.arraydata_locations_for_coverage(coverage) range_type = coverage.range_type bands = list(range_type) subsets = params.subsets if subsets: subsets.srid # this automatically checks the validity # create and configure map object map_ = self.create_map() env = {} for data_location in data_locations: env.update(data_location.env) gdal.set_env(env, False) # configure outputformat native_format = self.get_native_format(coverage, data_locations) if native_format and get_format_by_mime(native_format) is None: native_format = "image/tiff" frmt = params.format or native_format if frmt is None: raise RenderException("Format could not be determined", "format") mime_type, frmt = split_format(frmt) imagemode = ms.gdalconst_to_imagemode(bands[0].data_type) time_stamp = datetime.now().strftime("%Y%m%d%H%M%S") basename = "%s_%s" % (coverage.identifier, time_stamp) of = create_outputformat(mime_type, frmt, imagemode, basename, getattr(params, "encoding_params", {})) map_.appendOutputFormat(of) map_.setOutputFormat(of) # TODO: use layer factory here layer = self.layer_for_coverage(coverage, native_format, params.version) map_.insertLayer(layer) connector = get_connector_by_test(coverage, data_locations) if not connector: raise OperationNotSupportedException( "Could not find applicable layer connector.", "coverage") try: connector.connect(coverage, data_locations, layer, {}) # create request object and dispatch it against the map request = ms.create_request( self.translate_params(params, range_type)) request.setParameter("format", mime_type) raw_result = ms.dispatch(map_, request) finally: # perform any required layer related cleanup connector.disconnect(coverage, data_locations, layer, {}) result_set = result_set_from_raw_data(raw_result) if params.version == Version(2, 0): mediatype = getattr(params, "mediatype", None) if mediatype in ("multipart/mixed", "multipart/related"): with vsi.TemporaryVSIFile.from_buffer(result_set[1].data) as f: ds = gdal.Open(f.name) grid = objects.Grid.from_gdal_dataset(ds) # get the output CRS definition crs = params.outputcrs or subsets.crs or 'imageCRS' if crs == 'imageCRS': crs = coverage.grid.coordinate_reference_system grid._coordinate_reference_system = crs origin = objects.Origin.from_gdal_dataset(ds) size = [ds.RasterXSize, ds.RasterYSize] range_type = coverage.range_type if params.rangesubset: range_type = range_type.subset(params.rangesubset) coverage._grid = grid coverage._origin = origin coverage._size = size coverage._range_type = range_type if isinstance(result_set[1].filename, binary_type): file_name = result_set[1].filename.decode() else: file_name = result_set[1].filename reference = 'cid:coverage/%s' % file_name encoder = WCS20EOXMLEncoder() if not isinstance(coverage, objects.Mosaic): tree = encoder.encode_rectified_dataset( coverage, getattr(params, "http_request", None), reference, mime_type, subsets.bounding_polygon(coverage) if subsets else None) else: tree = encoder.encode_rectified_stitched_mosaic( coverage, getattr(params, "http_request", None), reference, mime_type, subsets.bounding_polygon(coverage) if subsets else None) result_set[0] = ResultBuffer(encoder.serialize(tree), encoder.content_type) # "default" response return result_set
def __init__(self, dataset, use_nodata=True): if isinstance(dataset, basestring): dataset = gdal.Open(dataset) self.dataset = dataset self.use_nodata = use_nodata
def process(self, input_filename, output_filename, geo_reference=None, generate_metadata=True): # open the dataset and create an In-Memory Dataset as copy # to perform optimizations ds = create_mem_copy(gdal.Open(input_filename)) gt = ds.GetGeoTransform() footprint_wkt = None if not geo_reference: if gt == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0): # TODO: maybe use a better check raise ValueError("No geospatial reference for unreferenced " "dataset given.") else: logger.debug("Applying geo reference '%s'." % type(geo_reference).__name__) ds, footprint_wkt = geo_reference.apply(ds) # apply optimizations for optimization in self.get_optimizations(ds): logger.debug("Applying optimization '%s'." % type(optimization).__name__) try: new_ds = optimization(ds) if new_ds is not ds: # cleanup afterwards cleanup_temp(ds) ds = new_ds except: cleanup_temp(ds) raise # generate the footprint from the dataset if not footprint_wkt: logger.debug("Generating footprint.") footprint_wkt = self._generate_footprint_wkt(ds) # check that footprint is inside of extent of generated image # regenerate otherwise else: tmp_extent = getExtentFromRectifiedDS(ds) tmp_bbox = Polygon.from_bbox((tmp_extent[0], tmp_extent[1], tmp_extent[2], tmp_extent[3])) tmp_footprint = GEOSGeometry(footprint_wkt) if not tmp_bbox.contains(tmp_footprint): footprint_wkt = tmp_footprint.intersection(tmp_bbox).wkt if self.footprint_alpha: logger.debug("Applying optimization 'AlphaBandOptimization'.") opt = AlphaBandOptimization() opt(ds, footprint_wkt) output_filename = self.generate_filename(output_filename) logger.debug("Writing file to disc using options: %s." % ", ".join(self.format_selection.creation_options)) logger.debug("Metadata tags to be written: %s" % ", ".join(ds.GetMetadata_List("") or [])) # save the file to the disc driver = gdal.GetDriverByName(self.format_selection.driver_name) ds = driver.CreateCopy(output_filename, ds, options=self.format_selection.creation_options) for optimization in self.get_post_optimizations(ds): logger.debug("Applying post-optimization '%s'." % type(optimization).__name__) optimization(ds) # generate metadata if requested footprint = None if generate_metadata: normalized_space = Polygon.from_bbox((-180, -90, 180, 90)) non_normalized_space = Polygon.from_bbox((180, -90, 360, 90)) footprint = GEOSGeometry(footprint_wkt) #.intersection(normalized_space) outer = non_normalized_space.intersection(footprint) if len(outer): footprint = MultiPolygon( *map(lambda p: Polygon(*map(lambda ls: LinearRing(*map(lambda point: (point[0] - 360, point[1]), ls.coords )), tuple(p) )), (outer,) ) ).union(normalized_space.intersection(footprint)) else: if isinstance(footprint, Polygon): footprint = MultiPolygon(footprint) logger.info("Calculated Footprint: '%s'" % footprint.wkt) # use the provided footprint #geom = OGRGeometry(footprint_wkt) #exterior = [] #for x, y in geom.exterior_ring.tuple: # exterior.append(y); exterior.append(x) #polygon = [exterior] num_bands = ds.RasterCount # finally close the dataset and write it to the disc ds = None return PreProcessResult(output_filename, footprint, num_bands)
class NGEOPreProcessor(WMSPreProcessor): def __init__(self, format_selection, overviews=True, crs=None, bands=None, bandmode=RGB, footprint_alpha=False, color_index=False, palette_file=None, no_data_value=None, overview_resampling=None, overview_levels=None, overview_minsize=None, radiometric_interval_min=None, radiometric_interval_max=None, sieve_max_threshold=None, simplification_factor=None, temporary_directory=None): self.format_selection = format_selection self.overviews = overviews self.overview_resampling = overview_resampling self.overview_levels = overview_levels self.overview_minsize = overview_minsize self.crs = crs self.bands = bands self.bandmode = bandmode self.footprint_alpha = footprint_alpha self.color_index = color_index self.palette_file = palette_file self.no_data_value = no_data_value self.radiometric_interval_min = radiometric_interval_min self.radiometric_interval_max = radiometric_interval_max if sieve_max_threshold is not None: self.sieve_max_threshold = sieve_max_threshold else: self.sieve_max_threshold = 0 if simplification_factor is not None: self.simplification_factor = simplification_factor else: # default 2 * resolution == 2 pixels self.simplification_factor = 2 self.temporary_directory = temporary_directory def process(self, input_filename, output_filename, geo_reference=None, generate_metadata=True, merge_with=None, original_footprint=None): # open the dataset and create an In-Memory Dataset as copy # to perform optimizations ds = create_mem_copy(gdal.Open(input_filename)) gt = ds.GetGeoTransform() footprint_wkt = None if not geo_reference: if gt == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0): if ds.GetGCPCount() > 0: geo_reference = InternalGCPs() else: raise ValueError("No geospatial reference for " "unreferenced dataset given.") if geo_reference: logger.debug("Applying geo reference '%s'." % type(geo_reference).__name__) # footprint is always in EPSG:4326 ds, footprint_wkt = geo_reference.apply(ds) # apply optimizations for optimization in self.get_optimizations(ds): logger.debug("Applying optimization '%s'." % type(optimization).__name__) try: new_ds = optimization(ds) if new_ds is not ds: # cleanup afterwards cleanup_temp(ds) ds = new_ds except: cleanup_temp(ds) raise # generate the footprint from the dataset if not footprint_wkt: logger.debug("Generating footprint.") footprint_wkt = self._generate_footprint_wkt(ds) # check that footprint is inside of extent of generated image # regenerate otherwise else: tmp_extent = getExtentFromRectifiedDS(ds) tmp_bbox = Polygon.from_bbox( (tmp_extent[0], tmp_extent[1], tmp_extent[2], tmp_extent[3])) # transform image bbox to EPSG:4326 if necessary proj = ds.GetProjection() srs = osr.SpatialReference() try: srs.ImportFromWkt(proj) srs.AutoIdentifyEPSG() ptype = "PROJCS" if srs.IsProjected() else "GEOGCS" srid = int(srs.GetAuthorityCode(ptype)) if srid != '4326': out_srs = osr.SpatialReference() out_srs.ImportFromEPSG(4326) transform = osr.CoordinateTransformation(srs, out_srs) tmp_bbox2 = ogr.CreateGeometryFromWkt(tmp_bbox.wkt) tmp_bbox2.Transform(transform) tmp_bbox = GEOSGeometry(tmp_bbox2.ExportToWkt()) except (RuntimeError, TypeError), e: logger.warn("Projection: %s" % proj) logger.warn("Failed to identify projection's EPSG code." "%s: %s" % (type(e).__name__, str(e))) tmp_footprint = GEOSGeometry(footprint_wkt) if not tmp_bbox.contains(tmp_footprint): logger.debug("Re-generating footprint because not inside of " "generated image.") footprint_wkt = tmp_footprint.intersection(tmp_bbox).wkt if self.footprint_alpha: logger.debug("Applying optimization 'AlphaBandOptimization'.") opt = AlphaBandOptimization() opt(ds, footprint_wkt) output_filename = self.generate_filename(output_filename) if merge_with is not None: if original_footprint is None: raise ValueError( "Original footprint with to be merged image required.") original_ds = gdal.Open(merge_with, gdal.GA_Update) merger = GDALDatasetMerger([ GDALGeometryMaskMergeSource( original_ds, original_footprint, temporary_directory=self.temporary_directory), GDALGeometryMaskMergeSource( ds, footprint_wkt, temporary_directory=self.temporary_directory) ]) final_ds = merger.merge(output_filename, self.format_selection.driver_name, self.format_selection.creation_options) # cleanup previous file driver = original_ds.GetDriver() original_ds = None driver.Delete(merge_with) cleanup_temp(ds) else: logger.debug("Writing single file '%s' using options: %s." % (output_filename, ", ".join( self.format_selection.creation_options))) logger.debug("Metadata tags to be written: %s" % ", ".join(ds.GetMetadata_List("") or [])) # save the file to the disc driver = gdal.GetDriverByName(self.format_selection.driver_name) final_ds = driver.CreateCopy( output_filename, ds, options=self.format_selection.creation_options) # cleanup cleanup_temp(ds) for optimization in self.get_post_optimizations(final_ds): logger.debug("Applying post-optimization '%s'." % type(optimization).__name__) optimization(final_ds) # generate metadata if requested footprint = None if generate_metadata: normalized_space = Polygon.from_bbox((-180, -90, 180, 90)) non_normalized_space = Polygon.from_bbox((180, -90, 360, 90)) footprint = GEOSGeometry(footprint_wkt) outer = non_normalized_space.intersection(footprint) if len(outer): footprint = MultiPolygon(*map( lambda p: Polygon(*map( lambda ls: LinearRing(*map( lambda point: (point[0] - 360, point[1]), ls.coords )), tuple(p))), (outer, ))).union( normalized_space.intersection(footprint)) else: if isinstance(footprint, Polygon): footprint = MultiPolygon(footprint) if original_footprint: logger.debug("Merging footprint.") footprint = footprint.union(GEOSGeometry(original_footprint)) logger.debug("Calculated Footprint: '%s'" % footprint.wkt) num_bands = final_ds.RasterCount # finally close the dataset and write it to the disc final_ds = None return PreProcessResult(output_filename, footprint, num_bands)
def process(self, input_filename, output_filename, geo_reference=None, generate_metadata=True, merge_with=None, original_footprint=None): # open the dataset and create an In-Memory Dataset as copy # to perform optimizations ds = create_mem_copy(gdal.Open(input_filename)) gt = ds.GetGeoTransform() footprint_wkt = None if not geo_reference: if gt == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0): if ds.GetGCPCount() > 0: geo_reference = InternalGCPs() else: raise ValueError("No geospatial reference for " "unreferenced dataset given.") if geo_reference: logger.debug("Applying geo reference '%s'." % type(geo_reference).__name__) # footprint is always in EPSG:4326 ds, footprint_wkt = geo_reference.apply(ds) # apply optimizations for optimization in self.get_optimizations(ds): logger.debug("Applying optimization '%s'." % type(optimization).__name__) try: new_ds = optimization(ds) if new_ds is not ds: # cleanup afterwards cleanup_temp(ds) ds = new_ds except: cleanup_temp(ds) raise # generate the footprint from the dataset if not footprint_wkt: logger.debug("Generating footprint.") footprint_wkt = self._generate_footprint_wkt(ds) # check that footprint is inside of extent of generated image # regenerate otherwise else: tmp_extent = getExtentFromRectifiedDS(ds) tmp_bbox = Polygon.from_bbox( (tmp_extent[0], tmp_extent[1], tmp_extent[2], tmp_extent[3])) # transform image bbox to EPSG:4326 if necessary proj = ds.GetProjection() srs = osr.SpatialReference() try: srs.ImportFromWkt(proj) srs.AutoIdentifyEPSG() ptype = "PROJCS" if srs.IsProjected() else "GEOGCS" srid = int(srs.GetAuthorityCode(ptype)) if srid != '4326': out_srs = osr.SpatialReference() out_srs.ImportFromEPSG(4326) transform = osr.CoordinateTransformation(srs, out_srs) tmp_bbox2 = ogr.CreateGeometryFromWkt(tmp_bbox.wkt) tmp_bbox2.Transform(transform) tmp_bbox = GEOSGeometry(tmp_bbox2.ExportToWkt()) except (RuntimeError, TypeError), e: logger.warn("Projection: %s" % proj) logger.warn("Failed to identify projection's EPSG code." "%s: %s" % (type(e).__name__, str(e))) tmp_footprint = GEOSGeometry(footprint_wkt) if not tmp_bbox.contains(tmp_footprint): logger.debug("Re-generating footprint because not inside of " "generated image.") footprint_wkt = tmp_footprint.intersection(tmp_bbox).wkt
def handle_with_cache(self, cache, *args, **kwargs): metadata_component = MetadataComponent(env) datas = kwargs["data"] semantics = kwargs.get("semantics") metadatas = kwargs["metadata"] range_type_name = kwargs["range_type_name"] if range_type_name is None: raise CommandError("No range type name specified.") range_type = models.RangeType.objects.get(name=range_type_name) metadata_keys = set(( "identifier", "extent", "size", "projection", "footprint", "begin_time", "end_time", "coverage_type", )) all_data_items = [] retrieved_metadata = {} retrieved_metadata.update( self._get_overrides(**kwargs) ) for metadata in metadatas: storage, package, format, location = self._get_location_chain( metadata ) data_item = backends.DataItem( location=location, format=format or "", semantic="metadata", storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) with open(connect(data_item, cache)) as f: content = f.read() reader = metadata_component.get_reader_by_test(content) if reader: values = reader.read(content) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): if key in metadata_keys: retrieved_metadata.setdefault(key, value) if len(datas) < 1: raise CommandError("No data files specified.") if semantics is None: # TODO: check corner cases. # e.g: only one data item given but multiple bands in range type # --> bands[1:<bandnum>] if len(datas) == 1: if len(range_type) == 1: semantics = ["bands[1]"] else: semantics = ["bands[1:%d]" % len(range_type)] else: semantics = ["bands[%d]" % i for i in range(len(datas))] for data, semantic in zip(datas, semantics): storage, package, format, location = self._get_location_chain(data) data_item = backends.DataItem( location=location, format=format or "", semantic=semantic, storage=storage, package=package, ) data_item.full_clean() data_item.save() all_data_items.append(data_item) try: ds = gdal.Open(connect(data_item, cache)) except: with open(connect(data_item, cache)) as f: ds = f.read() reader = metadata_component.get_reader_by_test(ds) if reader: values = reader.read(ds) format = values.pop("format", None) if format: data_item.format = format data_item.full_clean() data_item.save() for key, value in values.items(): retrieved_metadata.setdefault(key, value) ds = None if len(metadata_keys - set(retrieved_metadata.keys())): raise CommandError( "Missing metadata keys %s." % ", ".join(metadata_keys - set(retrieved_metadata.keys())) ) # replace any already registered dataset if kwargs["replace"]: try: # get a list of all collections the coverage was in. coverage = models.Coverage.objects.get( identifier=retrieved_metadata["identifier"] ) additional_ids = [ c.identifier for c in models.Collection.objects.filter( eo_objects__in=[coverage.pk] ) ] coverage.delete() self.print_msg( "Replacing previous dataset '%s'." % retrieved_metadata["identifier"] ) collection_ids = kwargs["collection_ids"] or [] for identifier in additional_ids: if identifier not in collection_ids: collection_ids.append(identifier) kwargs["collection_ids"] = collection_ids except models.Coverage.DoesNotExist: self.print_msg( "Could not replace previous dataset '%s'." % retrieved_metadata["identifier"] ) try: coverage_type = retrieved_metadata["coverage_type"] # TODO: allow types of different apps if len(coverage_type.split(".")) > 1: module_name, _, coverage_type = coverage_type.rpartition(".") module = import_module(module_name) CoverageType = getattr(module, coverage_type) else: CoverageType = getattr(models, coverage_type) except AttributeError: raise CommandError( "Type '%s' is not supported." % retrieved_metadata["coverage_type"] ) try: coverage = CoverageType() coverage.range_type = range_type proj = retrieved_metadata.pop("projection") if isinstance(proj, int): retrieved_metadata["srid"] = proj else: definition, format = proj # Try to identify the SRID from the given input try: sr = osr.SpatialReference(definition, format) retrieved_metadata["srid"] = sr.srid except Exception, e: prj = models.Projection.objects.get( format=format, definition=definition ) retrieved_metadata["projection"] = prj # TODO: bug in models for some coverages for key, value in retrieved_metadata.items(): setattr(coverage, key, value) coverage.visible = kwargs["visible"] coverage.full_clean() coverage.save() for data_item in all_data_items: data_item.dataset = coverage data_item.full_clean() data_item.save() # link with collection(s) if kwargs["collection_ids"]: ignore_missing_collection = kwargs["ignore_missing_collection"] call_command("eoxs_collection_link", collection_ids=kwargs["collection_ids"], add_ids=[coverage.identifier], ignore_missing_collection=ignore_missing_collection )
def register(self, metadata_locations, mask_locations, package_path, overrides, identifier_template=None, type_name=None, extended_metadata=True, discover_masks=True, discover_browses=True, discover_metadata=True, replace=False, simplify_footprint_tolerance=None): product_type = None if type_name: product_type = models.ProductType.objects.get(name=type_name) component = ProductMetadataComponent() browse_handles = [] mask_locations = mask_locations or [] metadata = {} package = None if package_path: handler = get_handler_by_test(package_path) if not handler: raise RegistrationError('Storage %r is not supported' % package_path) package, _ = backends.Storage.objects.get_or_create( url=package_path, storage_type=handler.name) if discover_masks or discover_browses or discover_metadata: collected_metadata = component.collect_package_metadata( package, handler) if discover_metadata: metadata.update(collected_metadata) if discover_browses: browse_handles.extend([ (browse_type, package_path, browse_path) for browse_type, browse_path in metadata.pop( 'browses', []) ]) if discover_masks: mask_locations.extend([(mask_type, package_path, mask_path) for mask_type, mask_path in metadata.pop('mask_files', [])]) mask_locations.extend([ (mask_type, geometry) for mask_type, geometry in metadata.pop('masks', []) ]) metadata_items = [ models.MetaDataItem(location=location[-1], storage=resolve_storage(location[:-1])) for location in metadata_locations ] new_metadata = {} for metadata_item in reversed(metadata_items): new_metadata.update( self._read_product_metadata(component, metadata_item)) mask_locations.extend(new_metadata.pop('masks', [])) metadata.update(new_metadata) metadata.update( dict((key, value) for key, value in overrides.items() if value is not None)) # apply overrides identifier = metadata.get('identifier') footprint = metadata.get('footprint') begin_time = metadata.get('begin_time') end_time = metadata.get('end_time') if identifier_template: identifier = identifier_template.format(metadata) metadata['identifier'] = identifier if simplify_footprint_tolerance is not None and footprint: footprint = footprint.simplify(simplify_footprint_tolerance, preserve_topology=True) replaced = False if replace: try: models.Product.objects.get(identifier=identifier).delete() replaced = True except models.Product.DoesNotExist: pass product = models.Product.objects.create( identifier=identifier, footprint=footprint, begin_time=begin_time, end_time=end_time, product_type=product_type, package=package, ) if extended_metadata and metadata: create_metadata(product, metadata) # register all masks for mask_handle in mask_locations: geometry = None storage = None location = '' if isinstance(mask_handle[1], GEOSGeometry): geometry = GEOSGeometry(mask_handle[1]) else: storage = resolve_storage(mask_handle[1:-1]) location = mask_handle[-1] try: mask_type = models.MaskType.objects.get( name=mask_handle[0], product_type=product_type) except models.MaskType.DoesNotExist: raise models.Mask.objects.create(product=product, mask_type=mask_type, storage=storage, location=location, geometry=geometry) # register all browses for browse_handle in browse_handles: browse_type = None if browse_handle[0]: # TODO: only browse types for that product type browse_type = models.BrowseType.objects.get( name=browse_handle[0], product_type=product_type) browse = models.Browse(product=product, location=browse_handle[-1], storage=resolve_storage( browse_handle[1:-1])) # Get a VSI handle for the browse to get the size, extent and CRS # via GDAL vsi_path = get_vsi_path(browse) ds = gdal.Open(vsi_path) browse.width = ds.RasterXSize browse.height = ds.RasterYSize browse.coordinate_reference_system = ds.GetProjection() extent = gdal.get_extent(ds) browse.min_x, browse.min_y, browse.max_x, browse.max_y = extent browse.full_clean() browse.save() for metadata_item in metadata_items: metadata_item.eo_object = product metadata_item.full_clean() metadata_item.save() return product, replaced