Example #1
0
    def read_from_file(self, filename):
        """Read and unpack vector data.

        It is assumed that the file contains only one layer with the
        pertinent features. Further it is assumed for the moment that
        all geometries are points.

        * A feature is a geometry and a set of attributes.
        * A geometry refers to location and can be point, line, polygon or
          combinations thereof.
        * The attributes or obtained through GetField()

        The full OGR architecture is documented at
        * http://www.gdal.org/ogr/ogr_arch.html
        * http://www.gdal.org/ogr/ogr_apitut.html

        Examples are at
        * danieljlewis.org/files/2010/09/basicpythonmap.pdf
        * http://invisibleroads.com/tutorials/gdal-shapefile-points-save.html
        * http://www.packtpub.com/article/geospatial-data-python-geometry

        Limitation of the Shapefile are documented in
        http://resources.esri.com/help/9.3/ArcGISDesktop/com/Gp_ToolRef/
        geoprocessing_tool_reference/
        geoprocessing_considerations_for_shapefile_output.htm
        """

        basename = os.path.splitext(filename)[0]

        # Look for any keywords
        self.keywords = read_keywords(basename + '.keywords')

        # FIXME (Ole): Should also look for style file to populate style_info

        # Determine name
        if 'title' in self.keywords:
            title = self.keywords['title']

            # Lookup internationalised title if available
            title = safe_tr(title)

            vectorname = title
        else:
            # Use basename without leading directories as name
            vectorname = os.path.split(basename)[-1]

        if self.name is None:
            self.name = vectorname
        self.filename = filename
        self.geometry_type = None  # In case there are no features

        fid = ogr.Open(filename)
        if fid is None:
            msg = 'Could not open %s' % filename
            raise ReadLayerError(msg)

        # Assume that file contains all data in one layer
        msg = 'Only one vector layer currently allowed'
        if fid.GetLayerCount() > 1 and self.sublayer is None:
            msg = ('WARNING: Number of layers in %s are %i. '
                   'Only the first layer will currently be '
                   'used. Specify sublayer when creating '
                   'the Vector if you wish to use a different layer.'
                   % (filename, fid.GetLayerCount()))
            LOGGER.warn(msg)
            # Why do we raise an exception if it is only a warning? TS
            raise ReadLayerError(msg)

        if self.sublayer is not None:
            layer = fid.GetLayerByName(self.sublayer)
        else:
            layer = fid.GetLayerByIndex(0)

        # Get spatial extent
        self.extent = layer.GetExtent()

        # Get projection
        p = layer.GetSpatialRef()
        self.projection = Projection(p)

        layer.ResetReading()

        # Extract coordinates and attributes for all features
        geometry = []
        data = []
        # Use feature iterator
        for feature in layer:
            # Record coordinates ordered as Longitude, Latitude
            G = feature.GetGeometryRef()
            if G is None:
                msg = ('Geometry was None in filename %s ' % filename)
                raise ReadLayerError(msg)
            else:
                self.geometry_type = G.GetGeometryType()
                if self.is_point_data:
                    geometry.append((G.GetX(), G.GetY()))
                elif self.is_line_data:
                    ring = get_ringdata(G)
                    geometry.append(ring)
                elif self.is_polygon_data:
                    polygon = get_polygondata(G)
                    geometry.append(polygon)
                elif self.is_multi_polygon_data:
                    try:
                        G = ogr.ForceToPolygon(G)
                    except:
                        msg = ('Got geometry type Multipolygon (%s) for '
                               'filename %s and could not convert it to '
                               'singlepart. However, you can use QGIS '
                               'functionality to convert multipart vector '
                               'data to singlepart (Vector -> Geometry Tools '
                               '-> Multipart to Singleparts and use the '
                               'resulting dataset.'
                               % (ogr.wkbMultiPolygon, filename))
                        raise ReadLayerError(msg)
                    else:
                        # Read polygon data as single part
                        self.geometry_type = ogr.wkbPolygon
                        polygon = get_polygondata(G)
                        geometry.append(polygon)
                else:
                    msg = ('Only point, line and polygon geometries are '
                           'supported. '
                           'Geometry type in filename %s '
                           'was %s.' % (filename,
                                        self.geometry_type))
                    raise ReadLayerError(msg)

            # Record attributes by name
            number_of_fields = feature.GetFieldCount()
            fields = {}
            for j in range(number_of_fields):
                name = feature.GetFieldDefnRef(j).GetName()

                # FIXME (Ole): Ascertain the type of each field?
                #              We need to cast each appropriately?
                #              This is issue #66
                #              (https://github.com/AIFDR/riab/issues/66)
                #feature_type = feature.GetFieldDefnRef(j).GetType()
                fields[name] = feature.GetField(j)

                    # We do this because there is NaN problem on windows
                    # NaN value must be converted to _pseudo_in to solve the
                    # problem. But, when InaSAFE read the file, it'll be
                    # converted back to NaN value, so that NaN in InaSAFE is a
                    # numpy.nan
                    # please check https://github.com/AIFDR/inasafe/issues/269
                    # for more information
                if fields[name] == _pseudo_inf:
                    fields[name] = float('nan')
                #print 'Field', name, feature_type, j, fields[name]

            data.append(fields)
        # Store geometry coordinates as a compact numeric array
        self.geometry = geometry
        self.data = data
Example #2
0
    def read_from_file(self, filename):
        """Read and unpack vector data.

        It is assumed that the file contains only one layer with the
        pertinent features. Further it is assumed for the moment that
        all geometries are points.

        * A feature is a geometry and a set of attributes.
        * A geometry refers to location and can be point, line, polygon or
          combinations thereof.
        * The attributes or obtained through GetField()

        The full OGR architecture is documented at
        * http://www.gdal.org/ogr/ogr_arch.html
        * http://www.gdal.org/ogr/ogr_apitut.html

        Examples are at
        * danieljlewis.org/files/2010/09/basicpythonmap.pdf
        * http://invisibleroads.com/tutorials/gdal-shapefile-points-save.html
        * http://www.packtpub.com/article/geospatial-data-python-geometry

        Limitation of the Shapefile are documented in
        http://resources.esri.com/help/9.3/ArcGISDesktop/com/Gp_ToolRef/
        geoprocessing_tool_reference/
        geoprocessing_considerations_for_shapefile_output.htm

        :param filename: a fully qualified location to the file
        :type filename: str

        :raises: ReadLayerError
        """

        base_name = os.path.splitext(filename)[0]

        # Look for any keywords
        self.keywords = read_keywords(base_name + '.keywords')

        # FIXME (Ole): Should also look for style file to populate style_info

        # Determine name
        if 'title' in self.keywords:
            title = self.keywords['title']

            # Lookup internationalised title if available
            title = safe_tr(title)

            vector_name = title
        else:
            # Use base_name without leading directories as name
            vector_name = os.path.split(base_name)[-1]

        if self.name is None:
            self.name = vector_name
        self.filename = filename
        self.geometry_type = None  # In case there are no features

        fid = ogr.Open(filename)
        if fid is None:
            msg = 'Could not open %s' % filename
            raise ReadLayerError(msg)

        # Assume that file contains all data in one layer
        msg = 'Only one vector layer currently allowed'
        if fid.GetLayerCount() > 1 and self.sublayer is None:
            msg = ('WARNING: Number of layers in %s are %i. '
                   'Only the first layer will currently be '
                   'used. Specify sublayer when creating '
                   'the Vector if you wish to use a different layer.' %
                   (filename, fid.GetLayerCount()))
            LOGGER.warn(msg)
            # Why do we raise an exception if it is only a warning? TS
            raise ReadLayerError(msg)

        if self.sublayer is not None:
            layer = fid.GetLayerByName(self.sublayer)
        else:
            layer = fid.GetLayerByIndex(0)

        # Get spatial extent
        self.extent = layer.GetExtent()

        # Get projection
        p = layer.GetSpatialRef()
        self.projection = Projection(p)

        layer.ResetReading()

        # Extract coordinates and attributes for all features
        geometry = []
        data = []
        # Use feature iterator
        for feature in layer:
            # Record coordinates ordered as Longitude, Latitude
            G = feature.GetGeometryRef()
            if G is None:
                msg = ('Geometry was None in filename %s ' % filename)
                raise ReadLayerError(msg)
            else:
                self.geometry_type = G.GetGeometryType()
                if self.is_point_data:
                    geometry.append((G.GetX(), G.GetY()))
                elif self.is_line_data:
                    ring = get_ring_data(G)
                    geometry.append(ring)
                elif self.is_polygon_data:
                    polygon = get_polygon_data(G)
                    geometry.append(polygon)
                elif self.is_multi_polygon_data:
                    try:
                        G = ogr.ForceToPolygon(G)
                    except:
                        msg = ('Got geometry type Multipolygon (%s) for '
                               'filename %s and could not convert it to '
                               'singlepart. However, you can use QGIS '
                               'functionality to convert multipart vector '
                               'data to singlepart (Vector -> Geometry Tools '
                               '-> Multipart to Singleparts and use the '
                               'resulting dataset.' %
                               (ogr.wkbMultiPolygon, filename))
                        raise ReadLayerError(msg)
                    else:
                        # Read polygon data as single part
                        self.geometry_type = ogr.wkbPolygon
                        polygon = get_polygon_data(G)
                        geometry.append(polygon)
                else:
                    msg = ('Only point, line and polygon geometries are '
                           'supported. '
                           'Geometry type in filename %s '
                           'was %s.' % (filename, self.geometry_type))
                    raise ReadLayerError(msg)

            # Record attributes by name
            number_of_fields = feature.GetFieldCount()
            fields = {}
            for j in range(number_of_fields):
                name = feature.GetFieldDefnRef(j).GetName()

                # FIXME (Ole): Ascertain the type of each field?
                #              We need to cast each appropriately?
                #              This is issue #66
                #              (https://github.com/AIFDR/riab/issues/66)
                #feature_type = feature.GetFieldDefnRef(j).GetType()
                fields[name] = feature.GetField(j)

                # We do this because there is NaN problem on windows
                # NaN value must be converted to _pseudo_in to solve the
                # problem. But, when InaSAFE read the file, it'll be
                # converted back to NaN value, so that NaN in InaSAFE is a
                # numpy.nan
                # please check https://github.com/AIFDR/inasafe/issues/269
                # for more information
                if fields[name] == _pseudo_inf:
                    fields[name] = float('nan')
                #print 'Field', name, feature_type, j, fields[name]

            data.append(fields)
        # Store geometry coordinates as a compact numeric array
        self.geometry = geometry
        self.data = data
Example #3
0
    def read_from_file(self, filename):
        """Read and unpack raster data
        """

        # Open data file for reading
        # File must be kept open, otherwise GDAL methods segfault.
        fid = self.fid = gdal.Open(filename, gdal.GA_ReadOnly)
        if fid is None:
            # As gdal doesn't return to us what the problem is we have to
            # figure it out ourselves. Maybe capture stderr?
            if not os.path.exists(filename):
                msg = 'Could not find file %s' % filename
            else:
                msg = ('File %s exists, but could not be read. '
                       'Please check if the file can be opened with '
                       'e.g. qgis or gdalinfo' % filename)
            raise ReadLayerError(msg)

        # Record raster metadata from file
        basename, ext = os.path.splitext(filename)

        # If file is ASCII, check that projection is around.
        # GDAL does not check this nicely, so it is worth an
        # error message
        if ext == '.asc':
            try:
                open(basename + '.prj')
            except IOError:
                msg = ('Projection file not found for %s. You must supply '
                       'a projection file with extension .prj' % filename)
                raise ReadLayerError(msg)

        # Look for any keywords
        self.keywords = read_keywords(basename + '.keywords')

        # Determine name
        if 'title' in self.keywords:
            title = self.keywords['title']

            # Lookup internationalised title if available
            title = safe_tr(title)

            rastername = title
        else:
            # Use basename without leading directories as name
            rastername = os.path.split(basename)[-1]

        if self.name is None:
            self.name = rastername
        self.filename = filename

        self.projection = Projection(self.fid.GetProjection())
        self.geotransform = self.fid.GetGeoTransform()
        self.columns = fid.RasterXSize
        self.rows = fid.RasterYSize
        self.number_of_bands = fid.RasterCount

        # Assume that file contains all data in one band
        msg = 'Only one raster band currently allowed'
        if self.number_of_bands > 1:
            msg = ('WARNING: Number of bands in %s are %i. '
                   'Only the first band will currently be '
                   'used.' % (filename, self.number_of_bands))
            # FIXME(Ole): Let us use python warnings here
            raise ReadLayerError(msg)

        # Get first band.
        band = self.band = fid.GetRasterBand(1)
        if band is None:
            msg = 'Could not read raster band from %s' % filename
            raise ReadLayerError(msg)
Example #4
0
    def read_from_file(self, filename):
        """Read and unpack raster data
        """

        # Open data file for reading
        # File must be kept open, otherwise GDAL methods segfault.
        fid = self.fid = gdal.Open(filename, gdal.GA_ReadOnly)
        if fid is None:
            # As gdal doesn't return to us what the problem is we have to
            # figure it out ourselves. Maybe capture stderr?
            if not os.path.exists(filename):
                msg = 'Could not find file %s' % filename
            else:
                msg = ('File %s exists, but could not be read. '
                       'Please check if the file can be opened with '
                       'e.g. qgis or gdalinfo' % filename)
            raise ReadLayerError(msg)

        # Record raster metadata from file
        basename, ext = os.path.splitext(filename)

        # If file is ASCII, check that projection is around.
        # GDAL does not check this nicely, so it is worth an
        # error message
        if ext == '.asc':
            try:
                open(basename + '.prj')
            except IOError:
                msg = ('Projection file not found for %s. You must supply '
                       'a projection file with extension .prj' % filename)
                raise ReadLayerError(msg)

        # Look for any keywords
        self.keywords = read_keywords(basename + '.keywords')

        # Determine name
        if 'title' in self.keywords:
            title = self.keywords['title']

            # Lookup internationalised title if available
            title = safe_tr(title)

            rastername = title
        else:
            # Use basename without leading directories as name
            rastername = os.path.split(basename)[-1]

        if self.name is None:
            self.name = rastername
        self.filename = filename

        self.projection = Projection(self.fid.GetProjection())
        self.geotransform = self.fid.GetGeoTransform()
        self.columns = fid.RasterXSize
        self.rows = fid.RasterYSize
        self.number_of_bands = fid.RasterCount

        # Assume that file contains all data in one band
        msg = 'Only one raster band currently allowed'
        if self.number_of_bands > 1:
            msg = ('WARNING: Number of bands in %s are %i. '
                   'Only the first band will currently be '
                   'used.' % (filename, self.number_of_bands))
            # FIXME(Ole): Let us use python warnings here
            raise ReadLayerError(msg)

        # Get first band.
        band = self.band = fid.GetRasterBand(1)
        if band is None:
            msg = 'Could not read raster band from %s' % filename
            raise ReadLayerError(msg)
Example #5
0
    def read_from_file(self, filename):
        """Read and unpack raster data
        """

        # Open data file for reading
        # File must be kept open, otherwise GDAL methods segfault.
        fid = self.fid = gdal.Open(filename, gdal.GA_ReadOnly)
        if fid is None:
            # As gdal doesn't return to us what the problem is we have to
            # figure it out ourselves. Maybe capture stderr?
            if not os.path.exists(filename):
                msg = 'Could not find file %s' % filename
            else:
                msg = ('File %s exists, but could not be read. '
                       'Please check if the file can be opened with '
                       'e.g. qgis or gdalinfo' % filename)
            raise ReadLayerError(msg)

        # Record raster metadata from file
        basename, ext = os.path.splitext(filename)

        # If file is ASCII, check that projection is around.
        # GDAL does not check this nicely, so it is worth an
        # error message
        if ext == '.asc':
            try:
                open(basename + '.prj')
            except IOError:
                msg = ('Projection file not found for %s. You must supply '
                       'a projection file with extension .prj' % filename)
                raise ReadLayerError(msg)

        # Look for any keywords
        self.keywords = read_keywords(basename + '.keywords')

        # Determine name
        if 'title' in self.keywords:
            title = self.keywords['title']

            # Lookup internationalised title if available
            title = safe_tr(title)

            rastername = title
        else:
            # Use basename without leading directories as name
            rastername = os.path.split(basename)[-1]

        if self.name is None:
            self.name = rastername
        self.filename = filename

        self.projection = Projection(self.fid.GetProjection())
        self.geotransform = self.fid.GetGeoTransform()
        self.columns = fid.RasterXSize
        self.rows = fid.RasterYSize
        self.number_of_bands = fid.RasterCount

        # Assume that file contains all data in one band
        msg = 'Only one raster band currently allowed'
        if self.number_of_bands > 1:
            msg = ('WARNING: Number of bands in %s are %i. '
                   'Only the first band will currently be '
                   'used.' % (filename, self.number_of_bands))
            # FIXME(Ole): Let us use python warnings here
            raise ReadLayerError(msg)

        # Get first band.
        band = self.band = fid.GetRasterBand(1)
        if band is None:
            msg = 'Could not read raster band from %s' % filename
            raise ReadLayerError(msg)

        # Force garbage collection to free up any memory we can (TS)
        gc.collect()

        # Read from raster file
        data = band.ReadAsArray()

        # Convert to double precision (issue #75)
        data = numpy.array(data, dtype=numpy.float64)

        # Self check
        M, N = data.shape
        msg = (
            'Dimensions of raster array do not match those of '
            'raster file %s' % self.filename)
        verify(M == self.rows, msg)
        verify(N == self.columns, msg)
        nodata = self.band.GetNoDataValue()
        if nodata is None:
            nodata = -9999

        if nodata is not numpy.nan:
            NaN = numpy.ones((M, N), numpy.float64) * numpy.nan
            data = numpy.where(data == nodata, NaN, data)

        self.data = data