def _write_shapefile_layer(layer_name, out_filename):
_log.info('writing layer {0} to {1} for {2}'.format(
layer_name,
out_filename,
self.resource.slug
))
if not os.path.exists(layer_name + '.prj'):
_log.warning("no projection file for {0}, assuming EPSG:4326".format(self.resource.slug))
with open(layer_name + '.prj', 'w') as prj:
prj.write(e4326)
saki = now()
sh.ogr2ogr(
'-explodecollections',
'-skipfailures',
'-append',
'-gt', '131072',
'-t_srs', 'epsg:3857',
'-f', 'SQLite',
'-dsco', 'SPATIALITE=YES',
out_filename, self.cache_path + '/' + layer_name + '.shp',
_out=stdout,
_err=stderr
)
ima = now()
_log.info("wrote shapefile layer {0} to {1} in {2}".format(layer_name, out_filename, ima-saki))
def update(path, kmz_url=KMZ_URL):
"""
Update the local conductivity region cache at *path* using the KMZ
file found at *kmz_url*.
"""
with SmartTempDir() as tmp_path:
# fetch KMZ file
local_fname = os.path.join(tmp_path, kmz_url.split('/')[-1])
logger.info('fetching {}'.format(kmz_url))
with open(local_fname,
'w') as out_fid, closing(urlopen(kmz_url)) as in_fid:
out_fid.write(in_fid.read())
# convert KMZ file
logger.info('converting KMZ file')
sh.ogr2ogr('ConductivityRegions',
'ConductivityRegions.kmz',
'-nlt',
'POLYGON',
f='ESRI Shapefile',
_cwd=tmp_path)
# save result
dst = os.path.join(path, 'ConductivityRegions')
shutil.copytree(os.path.join(tmp_path, 'ConductivityRegions'), dst)
assert os.path.isdir(dst)
return dst
def derive_dataset(cls,
title,
parent_page,
parent_dataresource,
owner=None):
from terrapyn.geocms.models import DataResource
from django.conf import settings
# create a new sqlite datasource
slug, srs, child_spec = parent_dataresource.driver_instance.get_rendering_parameters(
)
filename = child_spec['file']
new_filename = parent_dataresource.driver_instance.get_filename(
'sqlite').split('/')[-1]
new_filename = settings.MEDIA_ROOT + '/' + new_filename
sh.ogr2ogr('-f', 'SQLite', '-dsco', 'SPATIALITE=YES', '-t_srs',
'EPSG:3857', '-overwrite', '-skipfailures', new_filename,
filename)
ds = DataResource.objects.create(
title=title,
content=parent_dataresource.content,
parent=parent_page,
resource_file=new_filename,
driver='terrapyn.geocms.drivers.spatialite',
in_menus=[],
owner=owner)
return ds
def _write_shapefile_layer(layer_name, out_filename):
_log.info('writing layer {0} to {1} for {2}'.format(
layer_name, out_filename, self.resource.slug))
if not os.path.exists(layer_name + '.prj'):
_log.warning(
"no projection file for {0}, assuming EPSG:4326".format(
self.resource.slug))
with open(layer_name + '.prj', 'w') as prj:
prj.write(e4326)
saki = now()
sh.ogr2ogr('-explodecollections',
'-skipfailures',
'-append',
'-gt',
'131072',
'-t_srs',
'epsg:3857',
'-f',
'SQLite',
'-dsco',
'SPATIALITE=YES',
out_filename,
self.cache_path + '/' + layer_name + '.shp',
_out=stdout,
_err=stderr)
ima = now()
_log.info("wrote shapefile layer {0} to {1} in {2}".format(
layer_name, out_filename, ima - saki))
def _write_shapefile_layer(layer_name, out_filename):
_log.info("writing layer {0} to {1} for {2}".format(layer_name, out_filename, self.resource.slug))
if not os.path.exists(layer_name + ".prj"):
_log.warning("no projection file for {0}, assuming EPSG:4326".format(self.resource.slug))
with open(layer_name + ".prj", "w") as prj:
prj.write(e4326)
saki = now()
sh.ogr2ogr(
"-explodecollections",
"-skipfailures",
"-append",
"-gt",
"10384",
"-t_srs",
"epsg:3857",
"-f",
"SQLite",
"-dsco",
"SPATIALITE=YES",
out_filename,
self.cache_path + "/" + layer_name + ".shp",
_out=stdout,
_err=stderr,
)
ima = now()
_log.info("wrote shapefile layer {0} to {1} in {2}".format(layer_name, out_filename, ima - saki))
def derive_dataset(cls, title, parent_page, parent_dataresource, owner=None):
from ga_resources.models import DataResource
from django.conf import settings
# create a new sqlite datasource
slug, srs, child_spec = parent_dataresource.driver_instance.ready_data_resource()
filename = child_spec["file"]
new_filename = parent_dataresource.driver_instance.get_filename("sqlite").split("/")[-1]
new_filename = settings.MEDIA_ROOT + "/" + new_filename
sh.ogr2ogr(
"-f",
"SQLite",
"-dsco",
"SPATIALITE=YES",
"-t_srs",
"EPSG:3857",
"-overwrite",
"-skipfailures",
new_filename,
filename,
)
ds = DataResource.objects.create(
title=title,
content=parent_dataresource.content,
parent=parent_page,
resource_file=new_filename,
driver="ga_resources.drivers.spatialite",
in_menus=[],
owner=owner,
)
ds.driver_instance.ensure_local_file()
ds.driver_instance.compute_fields()
ds.driver_instance.ready_data_resource()
return ds
def run(self):
with self.input().open('r') as fobj:
coordinates = json.load(fobj)
label_filename = os.path.join(self.datapath, "input", "training",
"labels", self.filename + ".geojson")
dbname = utils.confparser.get("database", "dbname")
user = utils.confparser.get("database", "user")
password = utils.confparser.get("database", "password")
port = utils.confparser.get("database", "port")
host = utils.confparser.get("database", "host")
conn_string = ('PG:dbname={dbname} user={user} password={pw} '
'port={port} host={host}').format(dbname=dbname,
user=user,
pw=password,
port=port,
host=host)
ogr2ogr_args = [
'-f', 'PostGreSQL', conn_string,
os.path.join(self.datapath, "labels", self.filename + ".geojson"),
'-t_srs', "EPSG:{}".format(coordinates["srid"]), '-nln',
self.filename, '-overwrite'
]
with self.output().open("w") as fobj:
sh.ogr2ogr(ogr2ogr_args)
fobj.write(("ogr2ogr used file {} to insert OSM data "
"into {} database").format(label_filename, "tanzania"))
def derive_dataset(cls, title, parent_page, parent_dataresource, owner=None):
from terrapyn.geocms.models import DataResource
from django.conf import settings
# create a new sqlite datasource
slug, srs, child_spec = parent_dataresource.driver_instance.get_rendering_parameters()
filename = child_spec['file']
new_filename = parent_dataresource.driver_instance.get_filename('sqlite').split('/')[-1]
new_filename = settings.MEDIA_ROOT + '/' + new_filename
sh.ogr2ogr(
'-f', 'SQLite',
'-dsco', 'SPATIALITE=YES',
'-t_srs', 'EPSG:3857',
'-overwrite',
'-skipfailures',
new_filename, filename
)
ds = DataResource.objects.create(
title=title,
content=parent_dataresource.content,
parent=parent_page,
resource_file = new_filename,
driver='terrapyn.geocms.drivers.spatialite',
in_menus=[],
owner=owner
)
return ds
def process_data(self, source_file):
# convert any other kind of file to spatialite. this way the sqlite driver can be used with any OGR compatible
# file
_0, source_extension = os.path.splitext(source_file.name)
source_extension = source_extension.lower()
if source_extension.endswith('zip'):
out_filename, stdout, stderr = self._convert_zipfile(
os.path.join(settings.MEDIA_ROOT, source_file.name))
elif not source_extension.endswith('sqlite'):
_log.info('converting file for {0} into sqlite'.format(
self.resource.slug))
out_filename = self.get_filename('sqlite')
sh.rm('-f', out_filename)
if os.path.exists(out_filename):
os.unlink(out_filename)
stderr = StringIO()
stdout = StringIO()
saki = now()
sh.ogr2ogr('-explodecollections',
'-skipfailures',
'-overwrite',
'-gt',
'131072',
'-t_srs',
'epsg:3857',
'-f',
'SQLite',
'-dsco',
'SPATIALITE=YES',
out_filename,
os.path.join(settings.MEDIA_ROOT, source_file.name),
_out=stdout,
_err=stderr)
ima = now()
_log.info('write dataset {0} to {1} in {2}'.format(
source_file.name, out_filename, ima - saki))
source_file.close(
) # if the file was temporary, then this will delete it.
else:
return source_file, ""
return (File(open(out_filename),
name=self.resource.slug.split('/')[-1] + '.sqlite'),
"<<<INFO>>>\n{out}\n\n<<<ERRORS>>>\n{err}\n".format(
out=stdout.getvalue(), err=stderr.getvalue()))
def process_data(self, source_file):
# convert any other kind of file to spatialite. this way the sqlite driver can be used with any OGR compatible
# file
_0, source_extension = os.path.splitext(source_file.name)
source_extension = source_extension.lower()
if source_extension.endswith("zip"):
out_filename, stdout, stderr = self._convert_zipfile(os.path.join(settings.MEDIA_ROOT, source_file.name))
elif not source_extension.endswith("sqlite"):
_log.info("converting file for {0} into sqlite".format(self.resource.slug))
out_filename = self.get_filename("sqlite")
sh.rm("-f", out_filename)
if os.path.exists(out_filename):
os.unlink(out_filename)
stderr = StringIO()
stdout = StringIO()
saki = now()
sh.ogr2ogr(
"-explodecollections",
"-skipfailures",
"-overwrite",
"-gt",
"131072",
"-t_srs",
"epsg:3857",
"-f",
"SQLite",
"-dsco",
"SPATIALITE=YES",
out_filename,
os.path.join(settings.MEDIA_ROOT, source_file.name),
_out=stdout,
_err=stderr,
)
ima = now()
_log.info("write dataset {0} to {1} in {2}".format(source_file.name, out_filename, ima - saki))
source_file.close() # if the file was temporary, then this will delete it.
else:
return source_file, ""
return (
File(open(out_filename), name=self.resource.slug.split("/")[-1] + ".sqlite"),
"<<<INFO>>>\n{out}\n\n<<<ERRORS>>>\n{err}\n".format(out=stdout.getvalue(), err=stderr.getvalue()),
)
def _install_geo_partition_table(self, partition, table):
#
# Use ogr2ogr to copy.
#
import shlex
db = self.database
self.library.database.install_table(partition.get_table().vid, table.name)
args = [
"-t_srs EPSG:2771",
"-nlt PROMOTE_TO_MULTI",
"-nln {}".format(table.name),
"-progress ",
"-overwrite",
"-skipfailures",
"-f PostgreSQL",
("PG:'dbname={dbname} user={username} host={host} password={password}'"
.format(username=db.username, password=db.password,
host=db.server, dbname=db.dbname)),
partition.database.path,
"--config PG_USE_COPY YES"]
def err_output(line):
self.logger.error(line)
global count
count = 0
def out_output(c):
global count
count += 1
if count % 10 == 0:
pct = (int(count / 10)-1) * 20
if pct <= 100:
self.logger.log("Loading {}%".format(pct))
self.logger.log("Loading with: ogr2ogr {}".format(' '.join(args)))
# Need to shlex it b/c the "PG:" part gets bungled otherwise.
p = ogr2ogr(*shlex.split(' '.join(args)), _err=err_output, _out=out_output, _iter=True, _out_bufsize=0)
p.wait()
return
def load_ogr(self, partition, table_name):
#
# Use ogr2ogr to copy.
#
import shlex
from sh import ogr2ogr
p_vid = partition.identity.vid
d_vid = partition.identity.as_dataset().vid
a_table_name = self.augmented_table_name(d_vid, table_name)
args = [
"-t_srs EPSG:4326",
"-nlt PROMOTE_TO_MULTI",
"-nln {}".format(a_table_name),
"-progress ",
"-overwrite",
"-skipfailures",
] + self._ogr_args(partition)
def err_output(line):
self.logger.error(line)
global count
count = 0
def out_output(c):
global count
count += 1
if count % 10 == 0:
pct = (int(count / 10) - 1) * 20
if pct <= 100:
self.logger.info("Loading {}%".format(pct))
self.logger.info("Loading with: ogr2ogr {}".format(' '.join(args)))
# Need to shlex it b/c the "PG:" part gets bungled otherwise.
p = ogr2ogr(*shlex.split(' '.join(args)), _err=err_output, _out=out_output, _iter=True, _out_bufsize=0)
p.wait()
return a_table_name
def _convert_zipfile(self, source_filename):
# unzip file
# take stock of files unzipped
# if shapefiles exist, then look at unique base-names
# and create new layers in the output spatialite for each base-name
# and add projection files for each layer if they don't exist
_log.info('zipfile was designated for {0}, converting to sqlite'.format(self.resource.slug))
stdout = StringIO()
stderr = StringIO()
def _write_shapefile_layer(layer_name, out_filename):
_log.info('writing layer {0} to {1} for {2}'.format(
layer_name,
out_filename,
self.resource.slug
))
if not os.path.exists(layer_name + '.prj'):
_log.warning("no projection file for {0}, assuming EPSG:4326".format(self.resource.slug))
with open(layer_name + '.prj', 'w') as prj:
prj.write(e4326)
saki = now()
sh.ogr2ogr(
'-explodecollections',
'-skipfailures',
'-append',
'-gt', '131072',
'-t_srs', 'epsg:3857',
'-f', 'SQLite',
'-dsco', 'SPATIALITE=YES',
out_filename, self.cache_path + '/' + layer_name + '.shp',
_out=stdout,
_err=stderr
)
ima = now()
_log.info("wrote shapefile layer {0} to {1} in {2}".format(layer_name, out_filename, ima-saki))
e4326 = osr.SpatialReference()
e4326.ImportFromEPSG(4326)
e4326 = e4326.ExportToWkt()
out_filename = self.get_filename('sqlite')
if not os.path.exists(source_filename): # say it's stored in S3...
p, f = os.path.split(source_filename)
sh.mkdir('-p', p)
with open(source_filename, 'w') as out:
out.write(self.resource.original_file.read())
archive = ZipFile(source_filename)
names = archive.namelist()
names = filter(lambda x: ('.' in x) and (not x.startswith('__MACOSX')), sorted(names))
extensions = {os.path.splitext(name)[-1].lower() for name in names}
layer_name = self._layer_name(sorted(names)[0])
if '.shp' in extensions:
written = []
for name in names:
xtn = os.path.splitext(name)[-1]
this_layer_name = self._layer_name(name)
if os.path.exists(self.cache_path + '/' + this_layer_name + xtn):
os.unlink(self.cache_path + '/' + this_layer_name + xtn)
archive.extract(name, self.cache_path)
if name != (this_layer_name + xtn):
sh.mv(self.cache_path + '/' + name, self.cache_path + "/" + this_layer_name + xtn)
written.append(self.cache_path + '/' + this_layer_name + xtn)
if layer_name != this_layer_name:
_write_shapefile_layer(layer_name, out_filename)
layer_name = this_layer_name
_write_shapefile_layer(layer_name, out_filename)
for name in written:
os.unlink(name)
else:
sh.unzip(source_filename)
saki = now()
sh.ogr2ogr(
'-explodecollections',
'-skipfailures',
'-overwrite',
'-gt', '131072',
'-t_srs', 'epsg:3857',
'-f', 'SQLite',
'-dsco', 'SPATIALITE=YES',
out_filename,
source_filename.rsplit('.', 1)[0],
_out=stdout,
_err=stderr
)
ima = now()
_log.info('wrote dataset {0} to {1} in {2}'.format(
source_filename,
out_filename,
ima-saki
))
return out_filename, stdout, stderr
def compute_fields(self):
"""Other keyword args get passed in as a matter of course, like BBOX, time, and elevation, but this basic driver
ignores them"""
super(SpatialiteDriver, self).compute_fields()
if not hasattr(self, "src_ext") and self.resource.resource_file :
self.src_ext = self.resource.resource_file.split('.')[-1]
# convert any other kind of file to spatialite. this way the sqlite driver can be used with any OGR compatible
# file
if self.src_ext.endswith('zip'):
archive = ZipFile(self.cached_basename + self.src_ext)
projection_found = False
for name in archive.namelist():
xtn = name.split('.')[-1].lower()
if xtn in {'shp', 'shx', 'dbf', 'prj'} and "__MACOSX" not in name:
projection_found = projection_found or xtn == 'prj'
with open(self.cached_basename + '.' + xtn, 'wb') as fout:
with archive.open(name) as fin:
chunk = fin.read(65536)
while chunk:
fout.write(chunk)
chunk = fin.read(65536)
if not projection_found:
with open(self.cached_basename + '.prj', 'w') as f:
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
f.write(srs.ExportToWkt())
in_filename = self.get_filename('shp')
out_filename = self.get_filename('sqlite')
if os.path.exists(out_filename):
os.unlink(out_filename)
sh.ogr2ogr(
'-skipfailures',
'-t_srs', 'epsg:3857',
'-f', 'SQLite',
'-dsco', 'SPATIALITE=YES',
out_filename, in_filename
)
elif self.src_ext.endswith('gz'):
archive = TarFile(self.cached_basename + self.src_ext)
projection_found = False
for name in archive.getnames():
xtn = name.split('.')[-1].lower()
if xtn in {'shp', 'shx', 'dbf', 'prj'} and "__MACOSX" not in name:
projection_found = projection_found or xtn == 'prj'
with open(self.cached_basename + '.' + xtn, 'wb') as fout:
with archive.open(name) as fin:
chunk = fin.read(65536)
while chunk:
fout.write(chunk)
chunk = fin.read(65536)
if not projection_found:
with open(self.cached_basename + '.prj', 'w') as f:
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
f.write(srs.ExportToWkt())
in_filename = self.get_filename('shp')
out_filename = self.get_filename('sqlite')
if os.path.exists(out_filename):
os.unlink(out_filename)
sh.ogr2ogr(
'-skipfailures',
'-t_srs', 'epsg:3857',
'-f', 'SQLite',
'-dsco', 'SPATIALITE=YES',
out_filename, in_filename
)
elif not self.src_ext.endswith('sqlite'):
in_filename = self.get_filename(self.src_ext)
out_filename = self.get_filename('sqlite')
if os.path.exists(out_filename):
os.unlink(out_filename)
sh.ogr2ogr(
'-skipfailures',
'-t_srs', 'epsg:3857',
'-f', 'SQLite',
'-dsco', 'SPATIALITE=YES',
out_filename, in_filename
)
connection = self._connection()
table, geometry_field, _, _, srid, _ = connection.execute("select * from geometry_columns").fetchone() # grab the first layer with a geometry
self._srid = srid
dataframe = self.get_filename('dfx')
if os.path.exists(dataframe):
os.unlink(dataframe)
c = connection.cursor()
c.execute("select AsText(Extent(w.{geom_field})) from {table} as w".format(
geom_field=geometry_field,
table=table
))
try:
xmin, ymin, xmax, ymax = GEOSGeometry(c.fetchone()[0]).extent
except TypeError:
xmin = ymin = xmax = ymax = 0.0
crs = osr.SpatialReference()
crs.ImportFromEPSG(srid)
self.resource.native_srs = crs.ExportToProj4()
e4326 = osr.SpatialReference()
e4326.ImportFromEPSG(4326)
crx = osr.CoordinateTransformation(crs, e4326)
x04326, y04326, _ = crx.TransformPoint(xmin, ymin)
x14326, y14326, _ = crx.TransformPoint(xmax, ymax)
self.resource.bounding_box = Polygon.from_bbox((x04326, y04326, x14326, y14326))
self.resource.native_bounding_box = Polygon.from_bbox((xmin, ymin, xmax, ymax))
self.resource.three_d = False
self.resource.save()
def compute_fields(self):
"""Other keyword args get passed in as a matter of course, like BBOX, time, and elevation, but this basic driver
ignores them"""
super(SpatialiteDriver, self).compute_fields()
if not hasattr(self, "src_ext") and self.resource.resource_file:
self.src_ext = self.resource.resource_file.name.split(".")[-1]
elif not hasattr(self, "src_ext"):
self.src_ext = "sqlite"
# convert any other kind of file to spatialite. this way the sqlite driver can be used with any OGR compatible
# file
if self.src_ext.endswith("zip"):
archive = ZipFile(self.cached_basename + self.src_ext)
projection_found = False
for name in archive.namelist():
xtn = name.split(".")[-1].lower()
if xtn in {"shp", "shx", "dbf", "prj"} and "__MACOSX" not in name:
projection_found = projection_found or xtn == "prj"
with open(self.cached_basename + "." + xtn, "wb") as fout:
with archive.open(name) as fin:
chunk = fin.read(1024768)
while chunk:
fout.write(chunk)
chunk = fin.read(1024768)
if not projection_found:
with open(self.cached_basename + ".prj", "w") as f:
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
f.write(srs.ExportToWkt())
in_filename = self.get_filename("shp")
out_filename = self.get_filename("sqlite")
if os.path.exists(out_filename):
os.unlink(out_filename)
# ogr2ogr -skipfailures -overwrite -f SQLite -dsco USE_SPATIALITE=YES -lco OVERWRITE=YES -dsco OGR_SQLITE_SYNCHRONOUS=OFF -gt 131072 -t_srs epsg:3857
sh.ogr2ogr(
"-explodecollections",
"-skipfailures",
"-overwrite",
"-gt",
"131072",
"-t_srs",
"epsg:3857",
"-f",
"SQLite",
"-dsco",
"SPATIALITE=YES",
out_filename,
in_filename,
)
self.resource.resource_file = File(open(out_filename), name=self.resource.slug.split("/")[-1] + ".sqlite")
elif self.src_ext.endswith("gz"):
archive = TarFile(self.cached_basename + self.src_ext)
projection_found = False
for name in archive.getnames():
xtn = name.split(".")[-1].lower()
if xtn in {"shp", "shx", "dbf", "prj"} and "__MACOSX" not in name:
projection_found = projection_found or xtn == "prj"
with open(self.cached_basename + "." + xtn, "wb") as fout:
with archive.open(name) as fin:
chunk = fin.read(65536)
while chunk:
fout.write(chunk)
chunk = fin.read(65536)
if not projection_found:
with open(self.cached_basename + ".prj", "w") as f:
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
f.write(srs.ExportToWkt())
in_filename = self.get_filename("shp")
out_filename = self.get_filename("sqlite")
if os.path.exists(out_filename):
os.unlink(out_filename)
sh.ogr2ogr(
"-explodecollections",
"-skipfailures",
"-overwrite",
"-gt",
"131072",
"-t_srs",
"epsg:3857",
"-f",
"SQLite",
"-dsco",
"SPATIALITE=YES",
out_filename,
in_filename,
)
self.resource.resource_file = File(open(out_filename), name=self.resource.slug.split("/")[-1] + ".sqlite")
elif not self.src_ext.endswith("sqlite"):
in_filename = self.get_filename(self.src_ext)
out_filename = self.get_filename("sqlite")
if os.path.exists(out_filename):
os.unlink(out_filename)
sh.ogr2ogr(
"-explodecollections",
"-skipfailures",
"-overwrite",
"-gt",
"131072",
"-t_srs",
"epsg:3857",
"-f",
"SQLite",
"-dsco",
"SPATIALITE=YES",
out_filename,
in_filename,
)
self.resource.resource_file = File(open(out_filename), name=self.resource.slug.split("/")[-1] + ".sqlite")
connection = self._connection()
table, geometry_field, _, _, srid, _ = connection.execute(
"select * from geometry_columns"
).fetchone() # grab the first layer with a geometry
self._srid = srid if srid else 3857
dataframe = self.get_filename("dfx")
if os.path.exists(dataframe):
os.unlink(dataframe)
c = connection.cursor()
c.execute(
"select AsText(Extent(w.{geom_field})) from {table} as w".format(geom_field=geometry_field, table=table)
)
try:
xmin, ymin, xmax, ymax = GEOSGeometry(c.fetchone()[0]).extent
except TypeError:
xmin = ymin = xmax = ymax = 0.0
crs = osr.SpatialReference()
crs.ImportFromEPSG(srid)
self.resource.native_srs = crs.ExportToProj4()
c.execute("create index if not exists {table}_ogc_fid on {table} (OGC_FID)".format(table=table))
c.close()
e4326 = osr.SpatialReference()
e4326.ImportFromEPSG(4326)
crx = osr.CoordinateTransformation(crs, e4326)
x04326, y04326, _ = crx.TransformPoint(xmin, ymin)
x14326, y14326, _ = crx.TransformPoint(xmax, ymax)
self.resource.bounding_box = Polygon.from_bbox((x04326, y04326, x14326, y14326))
self.resource.native_bounding_box = Polygon.from_bbox((xmin, ymin, xmax, ymax))
self.resource.three_d = False
self.resource.save()
def _convert_zipfile(self, source_filename):
# unzip file
# take stock of files unzipped
# if shapefiles exist, then look at unique base-names
# and create new layers in the output spatialite for each base-name
# and add projection files for each layer if they don't exist
_log.info("zipfile was designated for {0}, converting to sqlite".format(self.resource.slug))
stdout = StringIO()
stderr = StringIO()
def _write_shapefile_layer(layer_name, out_filename):
_log.info("writing layer {0} to {1} for {2}".format(layer_name, out_filename, self.resource.slug))
if not os.path.exists(layer_name + ".prj"):
_log.warning("no projection file for {0}, assuming EPSG:4326".format(self.resource.slug))
with open(layer_name + ".prj", "w") as prj:
prj.write(e4326)
saki = now()
sh.ogr2ogr(
"-explodecollections",
"-skipfailures",
"-append",
"-gt",
"10384",
"-t_srs",
"epsg:3857",
"-f",
"SQLite",
"-dsco",
"SPATIALITE=YES",
out_filename,
self.cache_path + "/" + layer_name + ".shp",
_out=stdout,
_err=stderr,
)
ima = now()
_log.info("wrote shapefile layer {0} to {1} in {2}".format(layer_name, out_filename, ima - saki))
e4326 = osr.SpatialReference()
e4326.ImportFromEPSG(4326)
e4326 = e4326.ExportToWkt()
out_filename = self.get_filename("sqlite")
archive = ZipFile(source_filename)
names = archive.namelist()
names = filter(lambda x: ("." in x) and (not x.startswith("__MACOSX")), sorted(names))
extensions = {os.path.splitext(name)[-1].lower() for name in names}
layer_name = self._layer_name(sorted(names)[0])
if ".shp" in extensions:
written = []
for name in names:
xtn = os.path.splitext(name)[-1]
this_layer_name = self._layer_name(name)
if os.path.exists(self.cache_path + "/" + this_layer_name + xtn):
os.unlink(self.cache_path + "/" + this_layer_name + xtn)
archive.extract(name, self.cache_path)
if name != (this_layer_name + xtn):
sh.mv(self.cache_path + "/" + name, self.cache_path + "/" + this_layer_name + xtn)
written.append(self.cache_path + "/" + this_layer_name + xtn)
if layer_name != this_layer_name:
_write_shapefile_layer(layer_name, out_filename)
layer_name = this_layer_name
_write_shapefile_layer(layer_name, out_filename)
for name in written:
os.unlink(name)
else:
sh.unzip(source_filename)
saki = now()
sh.ogr2ogr(
"-explodecollections",
"-skipfailures",
"-overwrite",
"-gt",
"131072",
"-t_srs",
"epsg:3857",
"-f",
"SQLite",
"-dsco",
"SPATIALITE=YES",
out_filename,
source_filename.rsplit(".", 1)[0],
_out=stdout,
_err=stderr,
)
ima = now()
_log.info("wrote dataset {0} to {1} in {2}".format(source_filename, out_filename, ima - saki))
return out_filename, stdout, stderr
def _convert_zipfile(self, source_filename):
# unzip file
# take stock of files unzipped
# if shapefiles exist, then look at unique base-names
# and create new layers in the output spatialite for each base-name
# and add projection files for each layer if they don't exist
_log.info(
'zipfile was designated for {0}, converting to sqlite'.format(
self.resource.slug))
stdout = StringIO()
stderr = StringIO()
def _write_shapefile_layer(layer_name, out_filename):
_log.info('writing layer {0} to {1} for {2}'.format(
layer_name, out_filename, self.resource.slug))
if not os.path.exists(layer_name + '.prj'):
_log.warning(
"no projection file for {0}, assuming EPSG:4326".format(
self.resource.slug))
with open(layer_name + '.prj', 'w') as prj:
prj.write(e4326)
saki = now()
sh.ogr2ogr('-explodecollections',
'-skipfailures',
'-append',
'-gt',
'131072',
'-t_srs',
'epsg:3857',
'-f',
'SQLite',
'-dsco',
'SPATIALITE=YES',
out_filename,
self.cache_path + '/' + layer_name + '.shp',
_out=stdout,
_err=stderr)
ima = now()
_log.info("wrote shapefile layer {0} to {1} in {2}".format(
layer_name, out_filename, ima - saki))
e4326 = osr.SpatialReference()
e4326.ImportFromEPSG(4326)
e4326 = e4326.ExportToWkt()
out_filename = self.get_filename('sqlite')
if not os.path.exists(source_filename): # say it's stored in S3...
p, f = os.path.split(source_filename)
sh.mkdir('-p', p)
with open(source_filename, 'w') as out:
out.write(self.resource.original_file.read())
archive = ZipFile(source_filename)
names = archive.namelist()
names = filter(lambda x: ('.' in x) and (not x.startswith('__MACOSX')),
sorted(names))
extensions = {os.path.splitext(name)[-1].lower() for name in names}
layer_name = self._layer_name(sorted(names)[0])
if '.shp' in extensions:
written = []
for name in names:
xtn = os.path.splitext(name)[-1]
this_layer_name = self._layer_name(name)
if os.path.exists(self.cache_path + '/' + this_layer_name +
xtn):
os.unlink(self.cache_path + '/' + this_layer_name + xtn)
archive.extract(name, self.cache_path)
if name != (this_layer_name + xtn):
sh.mv(self.cache_path + '/' + name,
self.cache_path + "/" + this_layer_name + xtn)
written.append(self.cache_path + '/' + this_layer_name + xtn)
if layer_name != this_layer_name:
_write_shapefile_layer(layer_name, out_filename)
layer_name = this_layer_name
_write_shapefile_layer(layer_name, out_filename)
for name in written:
os.unlink(name)
else:
sh.unzip(source_filename)
saki = now()
sh.ogr2ogr('-explodecollections',
'-skipfailures',
'-overwrite',
'-gt',
'131072',
'-t_srs',
'epsg:3857',
'-f',
'SQLite',
'-dsco',
'SPATIALITE=YES',
out_filename,
source_filename.rsplit('.', 1)[0],
_out=stdout,
_err=stderr)
ima = now()
_log.info('wrote dataset {0} to {1} in {2}'.format(
source_filename, out_filename, ima - saki))
return out_filename, stdout, stderr
def compute_fields(self):
"""Other keyword args get passed in as a matter of course, like BBOX, time, and elevation, but this basic driver
ignores them"""
super(SpatialiteDriver, self).compute_fields()
if not hasattr(self, "src_ext") and self.resource.resource_file:
self.src_ext = self.resource.resource_file.split('.')[-1]
# convert any other kind of file to spatialite. this way the sqlite driver can be used with any OGR compatible
# file
if self.src_ext.endswith('zip'):
archive = ZipFile(self.cached_basename + self.src_ext)
projection_found = False
for name in archive.namelist():
xtn = name.split('.')[-1].lower()
if xtn in {'shp', 'shx', 'dbf', 'prj'
} and "__MACOSX" not in name:
projection_found = projection_found or xtn == 'prj'
with open(self.cached_basename + '.' + xtn, 'wb') as fout:
with archive.open(name) as fin:
chunk = fin.read(65536)
while chunk:
fout.write(chunk)
chunk = fin.read(65536)
if not projection_found:
with open(self.cached_basename + '.prj', 'w') as f:
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
f.write(srs.ExportToWkt())
in_filename = self.get_filename('shp')
out_filename = self.get_filename('sqlite')
if os.path.exists(out_filename):
os.unlink(out_filename)
sh.ogr2ogr('-skipfailures', '-t_srs', 'epsg:3857', '-f', 'SQLite',
'-dsco', 'SPATIALITE=YES', out_filename, in_filename)
elif self.src_ext.endswith('gz'):
archive = TarFile(self.cached_basename + self.src_ext)
projection_found = False
for name in archive.getnames():
xtn = name.split('.')[-1].lower()
if xtn in {'shp', 'shx', 'dbf', 'prj'
} and "__MACOSX" not in name:
projection_found = projection_found or xtn == 'prj'
with open(self.cached_basename + '.' + xtn, 'wb') as fout:
with archive.open(name) as fin:
chunk = fin.read(65536)
while chunk:
fout.write(chunk)
chunk = fin.read(65536)
if not projection_found:
with open(self.cached_basename + '.prj', 'w') as f:
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
f.write(srs.ExportToWkt())
in_filename = self.get_filename('shp')
out_filename = self.get_filename('sqlite')
if os.path.exists(out_filename):
os.unlink(out_filename)
sh.ogr2ogr('-skipfailures', '-t_srs', 'epsg:3857', '-f', 'SQLite',
'-dsco', 'SPATIALITE=YES', out_filename, in_filename)
elif not self.src_ext.endswith('sqlite'):
in_filename = self.get_filename(self.src_ext)
out_filename = self.get_filename('sqlite')
if os.path.exists(out_filename):
os.unlink(out_filename)
sh.ogr2ogr('-skipfailures', '-t_srs', 'epsg:3857', '-f', 'SQLite',
'-dsco', 'SPATIALITE=YES', out_filename, in_filename)
connection = self._connection()
table, geometry_field, _, _, srid, _ = connection.execute(
"select * from geometry_columns").fetchone(
) # grab the first layer with a geometry
self._srid = srid if srid else 3857
dataframe = self.get_filename('dfx')
if os.path.exists(dataframe):
os.unlink(dataframe)
c = connection.cursor()
c.execute(
"select AsText(Extent(w.{geom_field})) from {table} as w".format(
geom_field=geometry_field, table=table))
try:
xmin, ymin, xmax, ymax = GEOSGeometry(c.fetchone()[0]).extent
except TypeError:
xmin = ymin = xmax = ymax = 0.0
crs = osr.SpatialReference()
crs.ImportFromEPSG(srid)
self.resource.native_srs = crs.ExportToProj4()
e4326 = osr.SpatialReference()
e4326.ImportFromEPSG(4326)
crx = osr.CoordinateTransformation(crs, e4326)
x04326, y04326, _ = crx.TransformPoint(xmin, ymin)
x14326, y14326, _ = crx.TransformPoint(xmax, ymax)
self.resource.bounding_box = Polygon.from_bbox(
(x04326, y04326, x14326, y14326))
self.resource.native_bounding_box = Polygon.from_bbox(
(xmin, ymin, xmax, ymax))
self.resource.three_d = False
self.resource.save()