Example #1
0
        def _write_shapefile_layer(layer_name, out_filename):
            _log.info('writing layer {0} to {1} for {2}'.format(
                layer_name,
                out_filename,
                self.resource.slug
            ))

            if not os.path.exists(layer_name + '.prj'):
                _log.warning("no projection file for {0}, assuming EPSG:4326".format(self.resource.slug))
                with open(layer_name + '.prj', 'w') as prj:
                    prj.write(e4326)

            saki = now()
            sh.ogr2ogr(
                '-explodecollections',
                '-skipfailures',
                '-append',
                '-gt', '131072',
                '-t_srs', 'epsg:3857',
                '-f', 'SQLite',
                '-dsco', 'SPATIALITE=YES',
                out_filename, self.cache_path + '/' + layer_name + '.shp',
                _out=stdout,
                _err=stderr
            )
            ima = now()
            _log.info("wrote shapefile layer {0} to {1} in {2}".format(layer_name, out_filename, ima-saki))
Example #2
0
def update(path, kmz_url=KMZ_URL):
    """
    Update the local conductivity region cache at *path* using the KMZ
    file found at *kmz_url*.
    """
    with SmartTempDir() as tmp_path:
        # fetch KMZ file
        local_fname = os.path.join(tmp_path, kmz_url.split('/')[-1])
        logger.info('fetching {}'.format(kmz_url))
        with open(local_fname,
                  'w') as out_fid, closing(urlopen(kmz_url)) as in_fid:
            out_fid.write(in_fid.read())
        # convert KMZ file
        logger.info('converting KMZ file')
        sh.ogr2ogr('ConductivityRegions',
                   'ConductivityRegions.kmz',
                   '-nlt',
                   'POLYGON',
                   f='ESRI Shapefile',
                   _cwd=tmp_path)
        # save result
        dst = os.path.join(path, 'ConductivityRegions')
        shutil.copytree(os.path.join(tmp_path, 'ConductivityRegions'), dst)
    assert os.path.isdir(dst)
    return dst
Example #3
0
 def derive_dataset(cls,
                    title,
                    parent_page,
                    parent_dataresource,
                    owner=None):
     from terrapyn.geocms.models import DataResource
     from django.conf import settings
     # create a new sqlite datasource
     slug, srs, child_spec = parent_dataresource.driver_instance.get_rendering_parameters(
     )
     filename = child_spec['file']
     new_filename = parent_dataresource.driver_instance.get_filename(
         'sqlite').split('/')[-1]
     new_filename = settings.MEDIA_ROOT + '/' + new_filename
     sh.ogr2ogr('-f', 'SQLite', '-dsco', 'SPATIALITE=YES', '-t_srs',
                'EPSG:3857', '-overwrite', '-skipfailures', new_filename,
                filename)
     ds = DataResource.objects.create(
         title=title,
         content=parent_dataresource.content,
         parent=parent_page,
         resource_file=new_filename,
         driver='terrapyn.geocms.drivers.spatialite',
         in_menus=[],
         owner=owner)
     return ds
Example #4
0
        def _write_shapefile_layer(layer_name, out_filename):
            _log.info('writing layer {0} to {1} for {2}'.format(
                layer_name, out_filename, self.resource.slug))

            if not os.path.exists(layer_name + '.prj'):
                _log.warning(
                    "no projection file for {0}, assuming EPSG:4326".format(
                        self.resource.slug))
                with open(layer_name + '.prj', 'w') as prj:
                    prj.write(e4326)

            saki = now()
            sh.ogr2ogr('-explodecollections',
                       '-skipfailures',
                       '-append',
                       '-gt',
                       '131072',
                       '-t_srs',
                       'epsg:3857',
                       '-f',
                       'SQLite',
                       '-dsco',
                       'SPATIALITE=YES',
                       out_filename,
                       self.cache_path + '/' + layer_name + '.shp',
                       _out=stdout,
                       _err=stderr)
            ima = now()
            _log.info("wrote shapefile layer {0} to {1} in {2}".format(
                layer_name, out_filename, ima - saki))
Example #5
0
        def _write_shapefile_layer(layer_name, out_filename):
            _log.info("writing layer {0} to {1} for {2}".format(layer_name, out_filename, self.resource.slug))

            if not os.path.exists(layer_name + ".prj"):
                _log.warning("no projection file for {0}, assuming EPSG:4326".format(self.resource.slug))
                with open(layer_name + ".prj", "w") as prj:
                    prj.write(e4326)

            saki = now()
            sh.ogr2ogr(
                "-explodecollections",
                "-skipfailures",
                "-append",
                "-gt",
                "10384",
                "-t_srs",
                "epsg:3857",
                "-f",
                "SQLite",
                "-dsco",
                "SPATIALITE=YES",
                out_filename,
                self.cache_path + "/" + layer_name + ".shp",
                _out=stdout,
                _err=stderr,
            )
            ima = now()
            _log.info("wrote shapefile layer {0} to {1} in {2}".format(layer_name, out_filename, ima - saki))
Example #6
0
    def derive_dataset(cls, title, parent_page, parent_dataresource, owner=None):
        from ga_resources.models import DataResource
        from django.conf import settings

        # create a new sqlite datasource
        slug, srs, child_spec = parent_dataresource.driver_instance.ready_data_resource()
        filename = child_spec["file"]
        new_filename = parent_dataresource.driver_instance.get_filename("sqlite").split("/")[-1]
        new_filename = settings.MEDIA_ROOT + "/" + new_filename
        sh.ogr2ogr(
            "-f",
            "SQLite",
            "-dsco",
            "SPATIALITE=YES",
            "-t_srs",
            "EPSG:3857",
            "-overwrite",
            "-skipfailures",
            new_filename,
            filename,
        )
        ds = DataResource.objects.create(
            title=title,
            content=parent_dataresource.content,
            parent=parent_page,
            resource_file=new_filename,
            driver="ga_resources.drivers.spatialite",
            in_menus=[],
            owner=owner,
        )
        ds.driver_instance.ensure_local_file()
        ds.driver_instance.compute_fields()
        ds.driver_instance.ready_data_resource()
        return ds
Example #7
0
 def run(self):
     with self.input().open('r') as fobj:
         coordinates = json.load(fobj)
     label_filename = os.path.join(self.datapath, "input", "training",
                                   "labels", self.filename + ".geojson")
     dbname = utils.confparser.get("database", "dbname")
     user = utils.confparser.get("database", "user")
     password = utils.confparser.get("database", "password")
     port = utils.confparser.get("database", "port")
     host = utils.confparser.get("database", "host")
     conn_string = ('PG:dbname={dbname} user={user} password={pw} '
                    'port={port} host={host}').format(dbname=dbname,
                                                      user=user,
                                                      pw=password,
                                                      port=port,
                                                      host=host)
     ogr2ogr_args = [
         '-f', 'PostGreSQL', conn_string,
         os.path.join(self.datapath, "labels", self.filename + ".geojson"),
         '-t_srs', "EPSG:{}".format(coordinates["srid"]), '-nln',
         self.filename, '-overwrite'
     ]
     with self.output().open("w") as fobj:
         sh.ogr2ogr(ogr2ogr_args)
         fobj.write(("ogr2ogr used file {} to insert OSM data "
                     "into {} database").format(label_filename, "tanzania"))
Example #8
0
    def derive_dataset(cls, title, parent_page, parent_dataresource, owner=None):
        from terrapyn.geocms.models import DataResource
        from django.conf import settings
        # create a new sqlite datasource
        slug, srs, child_spec = parent_dataresource.driver_instance.get_rendering_parameters()
        filename = child_spec['file']
        new_filename = parent_dataresource.driver_instance.get_filename('sqlite').split('/')[-1]
        new_filename = settings.MEDIA_ROOT + '/' + new_filename
        sh.ogr2ogr(
            '-f', 'SQLite',
            '-dsco', 'SPATIALITE=YES',
            '-t_srs', 'EPSG:3857',
            '-overwrite',
            '-skipfailures',
            new_filename, filename
        )
        ds = DataResource.objects.create(
            title=title,
            content=parent_dataresource.content,
            parent=parent_page,
            resource_file = new_filename,
            driver='terrapyn.geocms.drivers.spatialite',
            in_menus=[],
            owner=owner

        )
        return ds
Example #9
0
    def process_data(self, source_file):
        # convert any other kind of file to spatialite.  this way the sqlite driver can be used with any OGR compatible
        # file
        _0, source_extension = os.path.splitext(source_file.name)
        source_extension = source_extension.lower()

        if source_extension.endswith('zip'):
            out_filename, stdout, stderr = self._convert_zipfile(
                os.path.join(settings.MEDIA_ROOT, source_file.name))
        elif not source_extension.endswith('sqlite'):
            _log.info('converting file for {0} into sqlite'.format(
                self.resource.slug))
            out_filename = self.get_filename('sqlite')
            sh.rm('-f', out_filename)

            if os.path.exists(out_filename):
                os.unlink(out_filename)

            stderr = StringIO()
            stdout = StringIO()

            saki = now()
            sh.ogr2ogr('-explodecollections',
                       '-skipfailures',
                       '-overwrite',
                       '-gt',
                       '131072',
                       '-t_srs',
                       'epsg:3857',
                       '-f',
                       'SQLite',
                       '-dsco',
                       'SPATIALITE=YES',
                       out_filename,
                       os.path.join(settings.MEDIA_ROOT, source_file.name),
                       _out=stdout,
                       _err=stderr)
            ima = now()

            _log.info('write dataset {0} to {1} in {2}'.format(
                source_file.name, out_filename, ima - saki))
            source_file.close(
            )  # if the file was temporary, then this will delete it.
        else:
            return source_file, ""

        return (File(open(out_filename),
                     name=self.resource.slug.split('/')[-1] + '.sqlite'),
                "<<<INFO>>>\n{out}\n\n<<<ERRORS>>>\n{err}\n".format(
                    out=stdout.getvalue(), err=stderr.getvalue()))
Example #10
0
    def process_data(self, source_file):
        # convert any other kind of file to spatialite.  this way the sqlite driver can be used with any OGR compatible
        # file
        _0, source_extension = os.path.splitext(source_file.name)
        source_extension = source_extension.lower()

        if source_extension.endswith("zip"):
            out_filename, stdout, stderr = self._convert_zipfile(os.path.join(settings.MEDIA_ROOT, source_file.name))
        elif not source_extension.endswith("sqlite"):
            _log.info("converting file for {0} into sqlite".format(self.resource.slug))
            out_filename = self.get_filename("sqlite")
            sh.rm("-f", out_filename)

            if os.path.exists(out_filename):
                os.unlink(out_filename)

            stderr = StringIO()
            stdout = StringIO()

            saki = now()
            sh.ogr2ogr(
                "-explodecollections",
                "-skipfailures",
                "-overwrite",
                "-gt",
                "131072",
                "-t_srs",
                "epsg:3857",
                "-f",
                "SQLite",
                "-dsco",
                "SPATIALITE=YES",
                out_filename,
                os.path.join(settings.MEDIA_ROOT, source_file.name),
                _out=stdout,
                _err=stderr,
            )
            ima = now()

            _log.info("write dataset {0} to {1} in {2}".format(source_file.name, out_filename, ima - saki))
            source_file.close()  # if the file was temporary, then this will delete it.
        else:
            return source_file, ""

        return (
            File(open(out_filename), name=self.resource.slug.split("/")[-1] + ".sqlite"),
            "<<<INFO>>>\n{out}\n\n<<<ERRORS>>>\n{err}\n".format(out=stdout.getvalue(), err=stderr.getvalue()),
        )
Example #11
0
    def _install_geo_partition_table(self, partition, table):
        #
        # Use ogr2ogr to copy. 
        #
        import shlex
        
        db = self.database
        
        self.library.database.install_table(partition.get_table().vid, table.name)
        
    
        args = [
        "-t_srs EPSG:2771",
        "-nlt PROMOTE_TO_MULTI",
        "-nln {}".format(table.name),
        "-progress ",
        "-overwrite",
        "-skipfailures",
        "-f PostgreSQL",
        ("PG:'dbname={dbname} user={username} host={host} password={password}'"
         .format(username=db.username, password=db.password, 
                    host=db.server, dbname=db.dbname)),
        partition.database.path,
        "--config PG_USE_COPY YES"]
        
        def err_output(line):
            self.logger.error(line)
        
        global count
        count = 0
        def out_output(c): 
            global count
            count +=  1
            if count % 10 == 0:
                pct =  (int(count / 10)-1) * 20
                if pct <= 100:
                    self.logger.log("Loading {}%".format(pct))  



        self.logger.log("Loading with: ogr2ogr {}".format(' '.join(args)))
        
        # Need to shlex it b/c the "PG:" part gets bungled otherwise. 
        p = ogr2ogr(*shlex.split(' '.join(args)),  _err=err_output, _out=out_output, _iter=True, _out_bufsize=0)
        p.wait()

        return
        

        
        
        
        
        
Example #12
0
    def load_ogr(self, partition, table_name):
        #
        # Use ogr2ogr to copy.
        #
        import shlex
        from sh import ogr2ogr

        p_vid = partition.identity.vid
        d_vid = partition.identity.as_dataset().vid

        a_table_name = self.augmented_table_name(d_vid, table_name)

        args = [
            "-t_srs EPSG:4326",
            "-nlt PROMOTE_TO_MULTI",
            "-nln {}".format(a_table_name),
            "-progress ",
            "-overwrite",
            "-skipfailures",
            ] + self._ogr_args(partition)

        def err_output(line):
            self.logger.error(line)

        global count
        count = 0


        def out_output(c):
            global count
            count += 1
            if count % 10 == 0:
                pct = (int(count / 10) - 1) * 20
                if pct <= 100:
                    self.logger.info("Loading {}%".format(pct))


        self.logger.info("Loading with: ogr2ogr {}".format(' '.join(args)))

        # Need to shlex it b/c the "PG:" part gets bungled otherwise.
        p = ogr2ogr(*shlex.split(' '.join(args)), _err=err_output, _out=out_output, _iter=True, _out_bufsize=0)
        p.wait()

        return a_table_name
Example #13
0
    def _convert_zipfile(self, source_filename):
        # unzip file
        # take stock of files unzipped
        # if shapefiles exist, then look at unique base-names
        # and create new layers in the output spatialite for each base-name
        # and add projection files for each layer if they don't exist

        _log.info('zipfile was designated for {0}, converting to sqlite'.format(self.resource.slug))

        stdout = StringIO()
        stderr = StringIO()

        def _write_shapefile_layer(layer_name, out_filename):
            _log.info('writing layer {0} to {1} for {2}'.format(
                layer_name,
                out_filename,
                self.resource.slug
            ))

            if not os.path.exists(layer_name + '.prj'):
                _log.warning("no projection file for {0}, assuming EPSG:4326".format(self.resource.slug))
                with open(layer_name + '.prj', 'w') as prj:
                    prj.write(e4326)

            saki = now()
            sh.ogr2ogr(
                '-explodecollections',
                '-skipfailures',
                '-append',
                '-gt', '131072',
                '-t_srs', 'epsg:3857',
                '-f', 'SQLite',
                '-dsco', 'SPATIALITE=YES',
                out_filename, self.cache_path + '/' + layer_name + '.shp',
                _out=stdout,
                _err=stderr
            )
            ima = now()
            _log.info("wrote shapefile layer {0} to {1} in {2}".format(layer_name, out_filename, ima-saki))


        e4326 = osr.SpatialReference()
        e4326.ImportFromEPSG(4326)
        e4326 = e4326.ExportToWkt()

        out_filename = self.get_filename('sqlite')
        if not os.path.exists(source_filename):  # say it's stored in S3...
            p, f = os.path.split(source_filename)
            sh.mkdir('-p', p)
            with open(source_filename, 'w') as out:
                out.write(self.resource.original_file.read())

        archive = ZipFile(source_filename)
        names = archive.namelist()
        names = filter(lambda x: ('.' in x) and (not x.startswith('__MACOSX')), sorted(names))
        extensions = {os.path.splitext(name)[-1].lower() for name in names}

        layer_name = self._layer_name(sorted(names)[0])
        if '.shp' in extensions:
            written = []
            for name in names:
                xtn = os.path.splitext(name)[-1]
                this_layer_name = self._layer_name(name)
                if os.path.exists(self.cache_path + '/' + this_layer_name + xtn):
                    os.unlink(self.cache_path + '/' + this_layer_name + xtn)

                archive.extract(name, self.cache_path)
                if name != (this_layer_name + xtn):
                    sh.mv(self.cache_path + '/' + name, self.cache_path + "/" + this_layer_name + xtn)

                written.append(self.cache_path + '/' + this_layer_name + xtn)

                if layer_name != this_layer_name:
                    _write_shapefile_layer(layer_name, out_filename)
                    layer_name = this_layer_name
            _write_shapefile_layer(layer_name, out_filename)
            for name in written:
                os.unlink(name)


        else:
            sh.unzip(source_filename)

            saki = now()
            sh.ogr2ogr(
                '-explodecollections',
                '-skipfailures',
                '-overwrite',
                '-gt', '131072',
                '-t_srs', 'epsg:3857',
                '-f', 'SQLite',
                '-dsco', 'SPATIALITE=YES',
                out_filename,
                source_filename.rsplit('.', 1)[0],
                _out=stdout,
                _err=stderr
            )
            ima = now()

            _log.info('wrote dataset {0} to {1} in {2}'.format(
                source_filename,
                out_filename,
                ima-saki
            ))

        return out_filename, stdout, stderr
Example #14
0
    def compute_fields(self):
        """Other keyword args get passed in as a matter of course, like BBOX, time, and elevation, but this basic driver
        ignores them"""
        
        super(SpatialiteDriver, self).compute_fields()

        if not hasattr(self, "src_ext") and self.resource.resource_file :
            self.src_ext = self.resource.resource_file.split('.')[-1]

        # convert any other kind of file to spatialite.  this way the sqlite driver can be used with any OGR compatible
        # file
        if self.src_ext.endswith('zip'):
            archive = ZipFile(self.cached_basename + self.src_ext)
            projection_found = False
            for name in archive.namelist():
                xtn = name.split('.')[-1].lower()
                if xtn in {'shp', 'shx', 'dbf', 'prj'} and "__MACOSX" not in name:
                    projection_found = projection_found or xtn == 'prj'
                    with open(self.cached_basename + '.' + xtn, 'wb') as fout:
                        with archive.open(name) as fin:
                            chunk = fin.read(65536)
                            while chunk:
                                fout.write(chunk)
                                chunk = fin.read(65536)

            if not projection_found:
                with open(self.cached_basename + '.prj', 'w') as f:
                    srs = osr.SpatialReference()
                    srs.ImportFromEPSG(4326)
                    f.write(srs.ExportToWkt())

            in_filename = self.get_filename('shp')
            out_filename = self.get_filename('sqlite')
            if os.path.exists(out_filename):
                os.unlink(out_filename)

            sh.ogr2ogr(
                '-skipfailures',
                '-t_srs', 'epsg:3857',
                '-f', 'SQLite',
                '-dsco', 'SPATIALITE=YES',
                out_filename, in_filename
            )
        elif self.src_ext.endswith('gz'):
            archive = TarFile(self.cached_basename + self.src_ext)
            projection_found = False
            for name in archive.getnames():
                xtn = name.split('.')[-1].lower()
                if xtn in {'shp', 'shx', 'dbf', 'prj'} and "__MACOSX" not in name:
                    projection_found = projection_found or xtn == 'prj'
                    with open(self.cached_basename + '.' + xtn, 'wb') as fout:
                        with archive.open(name) as fin:
                            chunk = fin.read(65536)
                            while chunk:
                                fout.write(chunk)
                                chunk = fin.read(65536)

            if not projection_found:
                with open(self.cached_basename + '.prj', 'w') as f:
                    srs = osr.SpatialReference()
                    srs.ImportFromEPSG(4326)
                    f.write(srs.ExportToWkt())

            in_filename = self.get_filename('shp')
            out_filename = self.get_filename('sqlite')
            if os.path.exists(out_filename):
                os.unlink(out_filename)

            sh.ogr2ogr(
                '-skipfailures',
                '-t_srs', 'epsg:3857',
                '-f', 'SQLite',
                '-dsco', 'SPATIALITE=YES',
                out_filename, in_filename
            )
        elif not self.src_ext.endswith('sqlite'):
            in_filename = self.get_filename(self.src_ext)
            out_filename = self.get_filename('sqlite')

            if os.path.exists(out_filename):
                os.unlink(out_filename)

            sh.ogr2ogr(
                '-skipfailures',
                '-t_srs', 'epsg:3857',
                '-f', 'SQLite',
                '-dsco', 'SPATIALITE=YES',
                out_filename, in_filename
            )

        connection = self._connection()
        table, geometry_field, _, _, srid, _ = connection.execute("select * from geometry_columns").fetchone() # grab the first layer with a geometry
        self._srid = srid

        dataframe = self.get_filename('dfx')
        if os.path.exists(dataframe):
            os.unlink(dataframe)

        c = connection.cursor()
        c.execute("select AsText(Extent(w.{geom_field})) from {table} as w".format(
            geom_field=geometry_field,
            table=table
        ))

        try:
            xmin, ymin, xmax, ymax = GEOSGeometry(c.fetchone()[0]).extent
        except TypeError:
            xmin = ymin = xmax = ymax = 0.0

        crs = osr.SpatialReference()
        crs.ImportFromEPSG(srid)
        self.resource.native_srs = crs.ExportToProj4()

        e4326 = osr.SpatialReference()
        e4326.ImportFromEPSG(4326)
        crx = osr.CoordinateTransformation(crs, e4326)
        x04326, y04326, _ = crx.TransformPoint(xmin, ymin)
        x14326, y14326, _ = crx.TransformPoint(xmax, ymax)
        self.resource.bounding_box = Polygon.from_bbox((x04326, y04326, x14326, y14326))
        self.resource.native_bounding_box = Polygon.from_bbox((xmin, ymin, xmax, ymax))
        self.resource.three_d = False

        self.resource.save()
Example #15
0
    def compute_fields(self):
        """Other keyword args get passed in as a matter of course, like BBOX, time, and elevation, but this basic driver
        ignores them"""

        super(SpatialiteDriver, self).compute_fields()

        if not hasattr(self, "src_ext") and self.resource.resource_file:
            self.src_ext = self.resource.resource_file.name.split(".")[-1]
        elif not hasattr(self, "src_ext"):
            self.src_ext = "sqlite"

        # convert any other kind of file to spatialite.  this way the sqlite driver can be used with any OGR compatible
        # file

        if self.src_ext.endswith("zip"):
            archive = ZipFile(self.cached_basename + self.src_ext)
            projection_found = False
            for name in archive.namelist():
                xtn = name.split(".")[-1].lower()
                if xtn in {"shp", "shx", "dbf", "prj"} and "__MACOSX" not in name:
                    projection_found = projection_found or xtn == "prj"
                    with open(self.cached_basename + "." + xtn, "wb") as fout:
                        with archive.open(name) as fin:
                            chunk = fin.read(1024768)
                            while chunk:
                                fout.write(chunk)
                                chunk = fin.read(1024768)

            if not projection_found:
                with open(self.cached_basename + ".prj", "w") as f:
                    srs = osr.SpatialReference()
                    srs.ImportFromEPSG(4326)
                    f.write(srs.ExportToWkt())

            in_filename = self.get_filename("shp")
            out_filename = self.get_filename("sqlite")
            if os.path.exists(out_filename):
                os.unlink(out_filename)

            # ogr2ogr -skipfailures -overwrite -f SQLite -dsco USE_SPATIALITE=YES -lco OVERWRITE=YES -dsco OGR_SQLITE_SYNCHRONOUS=OFF -gt 131072 -t_srs epsg:3857
            sh.ogr2ogr(
                "-explodecollections",
                "-skipfailures",
                "-overwrite",
                "-gt",
                "131072",
                "-t_srs",
                "epsg:3857",
                "-f",
                "SQLite",
                "-dsco",
                "SPATIALITE=YES",
                out_filename,
                in_filename,
            )
            self.resource.resource_file = File(open(out_filename), name=self.resource.slug.split("/")[-1] + ".sqlite")

        elif self.src_ext.endswith("gz"):
            archive = TarFile(self.cached_basename + self.src_ext)
            projection_found = False
            for name in archive.getnames():
                xtn = name.split(".")[-1].lower()
                if xtn in {"shp", "shx", "dbf", "prj"} and "__MACOSX" not in name:
                    projection_found = projection_found or xtn == "prj"
                    with open(self.cached_basename + "." + xtn, "wb") as fout:
                        with archive.open(name) as fin:
                            chunk = fin.read(65536)
                            while chunk:
                                fout.write(chunk)
                                chunk = fin.read(65536)

            if not projection_found:
                with open(self.cached_basename + ".prj", "w") as f:
                    srs = osr.SpatialReference()
                    srs.ImportFromEPSG(4326)
                    f.write(srs.ExportToWkt())

            in_filename = self.get_filename("shp")
            out_filename = self.get_filename("sqlite")
            if os.path.exists(out_filename):
                os.unlink(out_filename)

            sh.ogr2ogr(
                "-explodecollections",
                "-skipfailures",
                "-overwrite",
                "-gt",
                "131072",
                "-t_srs",
                "epsg:3857",
                "-f",
                "SQLite",
                "-dsco",
                "SPATIALITE=YES",
                out_filename,
                in_filename,
            )
            self.resource.resource_file = File(open(out_filename), name=self.resource.slug.split("/")[-1] + ".sqlite")

        elif not self.src_ext.endswith("sqlite"):
            in_filename = self.get_filename(self.src_ext)
            out_filename = self.get_filename("sqlite")

            if os.path.exists(out_filename):
                os.unlink(out_filename)

            sh.ogr2ogr(
                "-explodecollections",
                "-skipfailures",
                "-overwrite",
                "-gt",
                "131072",
                "-t_srs",
                "epsg:3857",
                "-f",
                "SQLite",
                "-dsco",
                "SPATIALITE=YES",
                out_filename,
                in_filename,
            )
            self.resource.resource_file = File(open(out_filename), name=self.resource.slug.split("/")[-1] + ".sqlite")

        connection = self._connection()
        table, geometry_field, _, _, srid, _ = connection.execute(
            "select * from geometry_columns"
        ).fetchone()  # grab the first layer with a geometry
        self._srid = srid if srid else 3857

        dataframe = self.get_filename("dfx")
        if os.path.exists(dataframe):
            os.unlink(dataframe)

        c = connection.cursor()
        c.execute(
            "select AsText(Extent(w.{geom_field})) from {table} as w".format(geom_field=geometry_field, table=table)
        )

        try:
            xmin, ymin, xmax, ymax = GEOSGeometry(c.fetchone()[0]).extent
        except TypeError:
            xmin = ymin = xmax = ymax = 0.0

        crs = osr.SpatialReference()
        crs.ImportFromEPSG(srid)
        self.resource.native_srs = crs.ExportToProj4()
        c.execute("create index if not exists {table}_ogc_fid on {table} (OGC_FID)".format(table=table))
        c.close()

        e4326 = osr.SpatialReference()
        e4326.ImportFromEPSG(4326)
        crx = osr.CoordinateTransformation(crs, e4326)
        x04326, y04326, _ = crx.TransformPoint(xmin, ymin)
        x14326, y14326, _ = crx.TransformPoint(xmax, ymax)
        self.resource.bounding_box = Polygon.from_bbox((x04326, y04326, x14326, y14326))
        self.resource.native_bounding_box = Polygon.from_bbox((xmin, ymin, xmax, ymax))
        self.resource.three_d = False

        self.resource.save()
Example #16
0
    def _convert_zipfile(self, source_filename):
        # unzip file
        # take stock of files unzipped
        # if shapefiles exist, then look at unique base-names
        # and create new layers in the output spatialite for each base-name
        # and add projection files for each layer if they don't exist

        _log.info("zipfile was designated for {0}, converting to sqlite".format(self.resource.slug))

        stdout = StringIO()
        stderr = StringIO()

        def _write_shapefile_layer(layer_name, out_filename):
            _log.info("writing layer {0} to {1} for {2}".format(layer_name, out_filename, self.resource.slug))

            if not os.path.exists(layer_name + ".prj"):
                _log.warning("no projection file for {0}, assuming EPSG:4326".format(self.resource.slug))
                with open(layer_name + ".prj", "w") as prj:
                    prj.write(e4326)

            saki = now()
            sh.ogr2ogr(
                "-explodecollections",
                "-skipfailures",
                "-append",
                "-gt",
                "10384",
                "-t_srs",
                "epsg:3857",
                "-f",
                "SQLite",
                "-dsco",
                "SPATIALITE=YES",
                out_filename,
                self.cache_path + "/" + layer_name + ".shp",
                _out=stdout,
                _err=stderr,
            )
            ima = now()
            _log.info("wrote shapefile layer {0} to {1} in {2}".format(layer_name, out_filename, ima - saki))

        e4326 = osr.SpatialReference()
        e4326.ImportFromEPSG(4326)
        e4326 = e4326.ExportToWkt()

        out_filename = self.get_filename("sqlite")
        archive = ZipFile(source_filename)
        names = archive.namelist()
        names = filter(lambda x: ("." in x) and (not x.startswith("__MACOSX")), sorted(names))
        extensions = {os.path.splitext(name)[-1].lower() for name in names}

        layer_name = self._layer_name(sorted(names)[0])
        if ".shp" in extensions:
            written = []
            for name in names:
                xtn = os.path.splitext(name)[-1]
                this_layer_name = self._layer_name(name)
                if os.path.exists(self.cache_path + "/" + this_layer_name + xtn):
                    os.unlink(self.cache_path + "/" + this_layer_name + xtn)

                archive.extract(name, self.cache_path)
                if name != (this_layer_name + xtn):
                    sh.mv(self.cache_path + "/" + name, self.cache_path + "/" + this_layer_name + xtn)

                written.append(self.cache_path + "/" + this_layer_name + xtn)

                if layer_name != this_layer_name:
                    _write_shapefile_layer(layer_name, out_filename)
                    layer_name = this_layer_name
            _write_shapefile_layer(layer_name, out_filename)
            for name in written:
                os.unlink(name)

        else:
            sh.unzip(source_filename)

            saki = now()
            sh.ogr2ogr(
                "-explodecollections",
                "-skipfailures",
                "-overwrite",
                "-gt",
                "131072",
                "-t_srs",
                "epsg:3857",
                "-f",
                "SQLite",
                "-dsco",
                "SPATIALITE=YES",
                out_filename,
                source_filename.rsplit(".", 1)[0],
                _out=stdout,
                _err=stderr,
            )
            ima = now()

            _log.info("wrote dataset {0} to {1} in {2}".format(source_filename, out_filename, ima - saki))

        return out_filename, stdout, stderr
Example #17
0
    def _convert_zipfile(self, source_filename):
        # unzip file
        # take stock of files unzipped
        # if shapefiles exist, then look at unique base-names
        # and create new layers in the output spatialite for each base-name
        # and add projection files for each layer if they don't exist

        _log.info(
            'zipfile was designated for {0}, converting to sqlite'.format(
                self.resource.slug))

        stdout = StringIO()
        stderr = StringIO()

        def _write_shapefile_layer(layer_name, out_filename):
            _log.info('writing layer {0} to {1} for {2}'.format(
                layer_name, out_filename, self.resource.slug))

            if not os.path.exists(layer_name + '.prj'):
                _log.warning(
                    "no projection file for {0}, assuming EPSG:4326".format(
                        self.resource.slug))
                with open(layer_name + '.prj', 'w') as prj:
                    prj.write(e4326)

            saki = now()
            sh.ogr2ogr('-explodecollections',
                       '-skipfailures',
                       '-append',
                       '-gt',
                       '131072',
                       '-t_srs',
                       'epsg:3857',
                       '-f',
                       'SQLite',
                       '-dsco',
                       'SPATIALITE=YES',
                       out_filename,
                       self.cache_path + '/' + layer_name + '.shp',
                       _out=stdout,
                       _err=stderr)
            ima = now()
            _log.info("wrote shapefile layer {0} to {1} in {2}".format(
                layer_name, out_filename, ima - saki))

        e4326 = osr.SpatialReference()
        e4326.ImportFromEPSG(4326)
        e4326 = e4326.ExportToWkt()

        out_filename = self.get_filename('sqlite')
        if not os.path.exists(source_filename):  # say it's stored in S3...
            p, f = os.path.split(source_filename)
            sh.mkdir('-p', p)
            with open(source_filename, 'w') as out:
                out.write(self.resource.original_file.read())

        archive = ZipFile(source_filename)
        names = archive.namelist()
        names = filter(lambda x: ('.' in x) and (not x.startswith('__MACOSX')),
                       sorted(names))
        extensions = {os.path.splitext(name)[-1].lower() for name in names}

        layer_name = self._layer_name(sorted(names)[0])
        if '.shp' in extensions:
            written = []
            for name in names:
                xtn = os.path.splitext(name)[-1]
                this_layer_name = self._layer_name(name)
                if os.path.exists(self.cache_path + '/' + this_layer_name +
                                  xtn):
                    os.unlink(self.cache_path + '/' + this_layer_name + xtn)

                archive.extract(name, self.cache_path)
                if name != (this_layer_name + xtn):
                    sh.mv(self.cache_path + '/' + name,
                          self.cache_path + "/" + this_layer_name + xtn)

                written.append(self.cache_path + '/' + this_layer_name + xtn)

                if layer_name != this_layer_name:
                    _write_shapefile_layer(layer_name, out_filename)
                    layer_name = this_layer_name
            _write_shapefile_layer(layer_name, out_filename)
            for name in written:
                os.unlink(name)

        else:
            sh.unzip(source_filename)

            saki = now()
            sh.ogr2ogr('-explodecollections',
                       '-skipfailures',
                       '-overwrite',
                       '-gt',
                       '131072',
                       '-t_srs',
                       'epsg:3857',
                       '-f',
                       'SQLite',
                       '-dsco',
                       'SPATIALITE=YES',
                       out_filename,
                       source_filename.rsplit('.', 1)[0],
                       _out=stdout,
                       _err=stderr)
            ima = now()

            _log.info('wrote dataset {0} to {1} in {2}'.format(
                source_filename, out_filename, ima - saki))

        return out_filename, stdout, stderr
Example #18
0
    def compute_fields(self):
        """Other keyword args get passed in as a matter of course, like BBOX, time, and elevation, but this basic driver
        ignores them"""

        super(SpatialiteDriver, self).compute_fields()

        if not hasattr(self, "src_ext") and self.resource.resource_file:
            self.src_ext = self.resource.resource_file.split('.')[-1]

        # convert any other kind of file to spatialite.  this way the sqlite driver can be used with any OGR compatible
        # file
        if self.src_ext.endswith('zip'):
            archive = ZipFile(self.cached_basename + self.src_ext)
            projection_found = False
            for name in archive.namelist():
                xtn = name.split('.')[-1].lower()
                if xtn in {'shp', 'shx', 'dbf', 'prj'
                           } and "__MACOSX" not in name:
                    projection_found = projection_found or xtn == 'prj'
                    with open(self.cached_basename + '.' + xtn, 'wb') as fout:
                        with archive.open(name) as fin:
                            chunk = fin.read(65536)
                            while chunk:
                                fout.write(chunk)
                                chunk = fin.read(65536)

            if not projection_found:
                with open(self.cached_basename + '.prj', 'w') as f:
                    srs = osr.SpatialReference()
                    srs.ImportFromEPSG(4326)
                    f.write(srs.ExportToWkt())

            in_filename = self.get_filename('shp')
            out_filename = self.get_filename('sqlite')
            if os.path.exists(out_filename):
                os.unlink(out_filename)

            sh.ogr2ogr('-skipfailures', '-t_srs', 'epsg:3857', '-f', 'SQLite',
                       '-dsco', 'SPATIALITE=YES', out_filename, in_filename)
        elif self.src_ext.endswith('gz'):
            archive = TarFile(self.cached_basename + self.src_ext)
            projection_found = False
            for name in archive.getnames():
                xtn = name.split('.')[-1].lower()
                if xtn in {'shp', 'shx', 'dbf', 'prj'
                           } and "__MACOSX" not in name:
                    projection_found = projection_found or xtn == 'prj'
                    with open(self.cached_basename + '.' + xtn, 'wb') as fout:
                        with archive.open(name) as fin:
                            chunk = fin.read(65536)
                            while chunk:
                                fout.write(chunk)
                                chunk = fin.read(65536)

            if not projection_found:
                with open(self.cached_basename + '.prj', 'w') as f:
                    srs = osr.SpatialReference()
                    srs.ImportFromEPSG(4326)
                    f.write(srs.ExportToWkt())

            in_filename = self.get_filename('shp')
            out_filename = self.get_filename('sqlite')
            if os.path.exists(out_filename):
                os.unlink(out_filename)

            sh.ogr2ogr('-skipfailures', '-t_srs', 'epsg:3857', '-f', 'SQLite',
                       '-dsco', 'SPATIALITE=YES', out_filename, in_filename)
        elif not self.src_ext.endswith('sqlite'):
            in_filename = self.get_filename(self.src_ext)
            out_filename = self.get_filename('sqlite')

            if os.path.exists(out_filename):
                os.unlink(out_filename)

            sh.ogr2ogr('-skipfailures', '-t_srs', 'epsg:3857', '-f', 'SQLite',
                       '-dsco', 'SPATIALITE=YES', out_filename, in_filename)

        connection = self._connection()
        table, geometry_field, _, _, srid, _ = connection.execute(
            "select * from geometry_columns").fetchone(
            )  # grab the first layer with a geometry
        self._srid = srid if srid else 3857

        dataframe = self.get_filename('dfx')
        if os.path.exists(dataframe):
            os.unlink(dataframe)

        c = connection.cursor()
        c.execute(
            "select AsText(Extent(w.{geom_field})) from {table} as w".format(
                geom_field=geometry_field, table=table))

        try:
            xmin, ymin, xmax, ymax = GEOSGeometry(c.fetchone()[0]).extent
        except TypeError:
            xmin = ymin = xmax = ymax = 0.0

        crs = osr.SpatialReference()
        crs.ImportFromEPSG(srid)
        self.resource.native_srs = crs.ExportToProj4()

        e4326 = osr.SpatialReference()
        e4326.ImportFromEPSG(4326)
        crx = osr.CoordinateTransformation(crs, e4326)
        x04326, y04326, _ = crx.TransformPoint(xmin, ymin)
        x14326, y14326, _ = crx.TransformPoint(xmax, ymax)
        self.resource.bounding_box = Polygon.from_bbox(
            (x04326, y04326, x14326, y14326))
        self.resource.native_bounding_box = Polygon.from_bbox(
            (xmin, ymin, xmax, ymax))
        self.resource.three_d = False

        self.resource.save()