def osm_from_geofabrik(out_folder):
"""
Get all OSM Files in GeoFabrik
"""
import os
from gasp.to.web import get_file
from gasp.cons.osm import osm_files
from gasp.pyt.oss import mkdir
main_url = "http://download.geofabrik.de/{}/{}-latest.osm.pbf"
for c in osm_files:
if c == 'russia':
get_file(
"http://download.geofabrik.de/russia-latest.osm.pbf",
os.path.join(out_folder, "russia-latest.osm.pbf")
)
continue
# Create folder for that continent
cf = mkdir(os.path.join(out_folder, c))
# Download files of every continent
for _c in osm_files[c]:
get_file(main_url.format(c, _c), os.path.join(
cf,
"{}-latest.osm.pbf".format(_c.replace('/', '_'))
), useWget=True)
return out_folder
def def_prj(shp, epsg=None, template=None, api='ogr'):
"""
Create/Replace the prj file of a ESRI Shapefile
API options:
* ogr;
* epsgio;
"""
import os
import shutil
prj_file = os.path.join(os.path.dirname(shp), '{}.prj'.format(
os.path.splitext(os.path.basename(shp))[0]
))
if api == 'ogr':
if epsg and not template:
s = osr.SpatialReference()
s.ImportFromEPSG(int(epsg))
s.MorphToESRI()
prj = open(prj_file, 'w')
prj.write(s.ExportToWkt())
prj.close()
return prj_file
elif not epsg and template:
prj_template = '{}.prj'.format(
os.path.splitext(os.path.basename(template))[0]
)
if not os.path.exists(prj_template):
return 0
try:
os.remove(prj_file)
shutil.copyfile(prj_template, prj_file)
except:
shutil.copyfile(prj_template, prj_file)
elif api == 'epsgio':
if not epsg:
raise ValueError((
"TO use epsgio option, epsg parameter must be given"
))
from gasp.to.web import get_file
from gasp.pyt.oss import del_file
url = 'https://epsg.io/{}.wkt?download'
if os.path.exists(prj_file):
# Delete file
del_file(prj_file)
# Get new prj
get_file(url.format(str(epsg)), prj_file)
return prj_file
def get_capabilities():
"""
Get GetCapabilities XML Data
"""
import os
import xmltodict
from gasp.to.web import get_file
from gasp.cons.gsrv import con_gsrv
from gasp.pyt.char import random_str
from gasp.pyt.oss import del_file
conparam = con_gsrv()
url = ("{}://{}:{}/geoserver/wms?request=GetCapabilities"
"&service=WMS&version=1.1.1").format(conparam["PROTOCOL"],
conparam["HOST"],
conparam["PORT"])
xml = get_file(
url,
os.path.join(os.path.dirname(os.path.abspath(__file__)),
random_str(10) + '.xml'))
with open(xml) as xmlf:
xmld = xmltodict.parse(xmlf.read())
del_file(xml)
return xmld
def osm_countries(out_shp):
"""
Get the boundary representing areas with OSM data in every
OSM PBF country file in GeoFabrik
"""
import os
import codecs
import pandas as pd
import geopandas as gpd
from shapely.geometry import MultiPolygon, Polygon
from gasp.cons.osm import osm_files
from gasp.to.web import get_file
from gasp.gt.toshp import df_to_shp
from gasp.pyt.oss import del_file
url = 'http://download.geofabrik.de/{}/{}.poly'
url_russia = 'http://download.geofabrik.de/russia.poly'
countries_boundaries = []
for c in osm_files:
for _c in osm_files[c]:
# Get poly file
ff = get_file(
url.format(c, _c) if c != 'russia' else url_russia,
os.path.join(
os.path.dirname(out_shp),
"{}_{}.poly".format(c, _c.replace('/', '_'))
if c != 'russia' else "russia.poly"))
# Get Polygon Coordinates
in_ring = False
coords = []
with open(ff, 'r') as txt:
i = 0
for l in txt.readlines():
if i == 0:
# first line is junk
i += 1
continue
elif i == 1:
# second line is the first polygon ring.
coords.append([[], []])
ring = coords[-1][0]
in_ring = True
i += 1
elif in_ring and l.strip() == 'END':
# we are at the end of a ring, perhaps with more to come.
in_ring = False
elif in_ring:
# we are in a ring and picking up new coordinates.
pnt = l.strip().split(' ')
ring.append((float(pnt[0]), float(pnt[1])))
elif not in_ring and l.strip() == 'END':
# we are at the end of the whole polygon.
break
elif not in_ring and l.startswith('!'):
# we are at the start of a polygon part hole.
coords[-1][1].append([])
ring = coords[-1][1][-1]
in_ring = True
elif not in_ring:
# we are at the start of a polygon part.
coords.append([[], []])
ring = coords[-1][0]
in_ring = True
polygon = MultiPolygon(coords)
countries_boundaries.append([c, _c, polygon])
del_file(ff)
countries_boundaries = gpd.GeoDataFrame(pd.DataFrame(
countries_boundaries, columns=['continent', 'country', 'geometry']),
crs={'init': 'epsg:4326'},
geometry='geometry')
return df_to_shp(countries_boundaries, out_shp)
def download_by_boundary(input_boundary, folder_out, osm_name, epsg,
GetUrl=True, db_name=None, geomCol=None,
justOneFeature=None):
"""
Download data from OSM using a bounding box
"""
import os
from osgeo import ogr
from gasp.to.web import get_file
from gasp.pyt.oss import os_name
OS_NAME = os_name()
EXTENTS = []
if db_name and geomCol:
"""
Assuming input_boundary is a PostgreSQL Table
"""
from gasp.pyt import obj_to_lst
from gasp.gql.prop import tbl_ext
for t in obj_to_lst(input_boundary):
EXTENTS.append(tbl_ext(db_name, t, geomCol))
else:
if type(input_boundary) == dict:
if 'top' in input_boundary and 'bottom' in input_boundary \
and 'left' in input_boundary and 'right' in input_boundary:
EXTENTS.append([
input_boundary['left'],input_boundary['right'],
input_boundary['bottom'], input_boundary['top']
])
else:
raise ValueError((
'input_boundary is a dict but the keys are not correct. '
'Please use left, right, top and bottom as keys'
))
elif type(input_boundary) == list:
if len(input_boundary) == 4:
EXTENTS.append(input_boundary)
else:
raise ValueError((
'input boundary is a list with more than 4 objects. '
'The list should be like: '
'l = [left, right, bottom, top]'
))
elif type(input_boundary) == ogr.Geometry:
EXTENTS.append(input_boundary.GetEnvelope())
else:
# Assuming input boundary is a file
#Check if file exists
if not os.path.exists(input_boundary):
raise ValueError((
"Sorry, but the file {} does not exist inside the folder {}!"
).format(
os.path.basename(input_boundary),
os.path.dirname(input_boundary)
))
# Check if is a raster
from gasp.gt.prop.ff import check_isRaster
isRst = check_isRaster(input_boundary)
# Get EPSG
if not epsg:
from gasp.gt.prop.prj import get_epsg
epsg = get_epsg(input_boundary)
if isRst:
from gasp.gt.prop.rst import rst_ext
# Get raster extent
EXTENTS.append(rst_ext(input_boundary))
else:
from gasp.gt.prop.ff import drv_name
# Todo: check if it is shape
# Open Dataset
inSrc = ogr.GetDriverByName(drv_name(
input_boundary)).Open(input_boundary)
lyr = inSrc.GetLayer()
i = 1
for feat in lyr:
geom = feat.GetGeometryRef()
featext = geom.GetEnvelope()
EXTENTS.append(featext)
if justOneFeature:
break
if epsg != 4326:
from gasp.g.to import new_pnt
from gasp.g.prj import prj_ogrgeom
for i in range(len(EXTENTS)):
bottom_left = prj_ogrgeom(new_pnt(
EXTENTS[i][0], EXTENTS[i][2]), epsg, 4326)
top_right = prj_ogrgeom(new_pnt(
EXTENTS[i][1], EXTENTS[i][3]), epsg, 4326)
left , bottom = bottom_left.GetX(), bottom_left.GetY()
right, top = top_right.GetX() , top_right.GetY()
EXTENTS[i] = [left, right, bottom, top]
#url = "https://overpass-api.de/api/map?bbox={}"
url = "https://lz4.overpass-api.de/api/interpreter?bbox={}"
RESULTS = []
for e in range(len(EXTENTS)):
bbox_str = ','.join([str(p) for p in EXTENTS[e]])
if GetUrl:
RESULTS.append(url.format(bbox_str))
continue
if len(EXTENTS) == 1:
outOsm = os.path.join(folder_out, osm_name + '.xml')
else:
outOsm = os.path.join(folder_out, "{}_{}.xml".format(osm_name, str(e)))
osm_file = get_file(
url.format(bbox_str), outOsm,
useWget=None if OS_NAME == 'Windows' else None
)
RESULTS.append(osm_file)
return RESULTS[0] if len(RESULTS) == 1 else RESULTS