def get_ext(inFile, outEpsg=None):
"""
Get Extent of any GIS Data
return None if inFile is not a GIS File
"""
from glass.g.prop import check_isRaster, check_isShp
if check_isRaster(inFile):
from glass.g.prop.rst import rst_ext
extent = rst_ext(inFile)
else:
if check_isShp(inFile):
from glass.g.prop.feat import get_ext as gext
extent = gext(inFile)
else:
return None
if outEpsg:
from glass.g.prop.prj import get_epsg
fileEpsg = get_epsg(inFile)
if not fileEpsg:
raise ValueError('cannot get EPSG of input file')
if fileEpsg != outEpsg:
from glass.g.gobj import new_pnt
from glass.g.prj.obj import prj_ogrgeom
bt_left = prj_ogrgeom(new_pnt(extent[0], extent[2]),
fileEpsg,
outEpsg,
api='ogr' if outEpsg != 4326 else 'shapely')
top_right = prj_ogrgeom(
new_pnt(extent[1], extent[3]),
fileEpsg,
outEpsg,
api='ogr' if outEpsg != 4326 else 'shapely')
left, bottom = bt_left.GetX(), bt_left.GetY()
right, top = top_right.GetX(), top_right.GetY()
extent = [left, right, bottom, top]
return extent
def bf_prop(buffer_shp, epsg_in, isFile=None):
"""
Return the centroid and radius distance of one buffer geometry
Centroid X, Y in the units of the buffer_shp;
Radius in meters.
Object return will be something like this:
o = {
'X': x_value,
'Y': y_value,
'R': radius_value
}
"""
from glass.g.prop.feat import get_cntr_bnd
if isFile:
from glass.g.tbl.filter import geom_by_idx
BUFFER_GEOM = ogr.CreateGeometryFromWkt(geom_by_idx(buffer_shp, 0))
else:
BUFFER_GEOM = ogr.CreateGeometryFromWkt(buffer_shp)
# Get x_center, y_center and dist from polygon geometry
# TODO: Besides 4326, we need to include also the others geographic systems
if int(epsg_in) == 4326:
from glass.g.prj.obj import prj_ogrgeom
BUFFER_GEOM_R = prj_ogrgeom(BUFFER_GEOM, epsg_in, 3857)
else:
BUFFER_GEOM_R = BUFFER_GEOM
dist = get_buffer_radius(BUFFER_GEOM_R.ExportToWkt(), isFile=None)
center = get_cntr_bnd(BUFFER_GEOM, isFile=None)
return {'X': center.GetX(), 'Y': center.GetY(), 'R': dist}
def coords_to_boundary(topLeft, lowerRight, epsg, outEpsg=None):
"""
Top Left and Lower Right to Boundary
"""
from osgeo import ogr
from glass.g.gobj import create_polygon
boundary_points = [(topLeft[0], topLeft[1]), (lowerRight[0], topLeft[1]),
(lowerRight[0], lowerRight[1]),
(topLeft[0], lowerRight[1]), (topLeft[0], topLeft[1])]
# Create polygon
polygon = create_polygon(boundary_points)
# Convert SRS if outEPSG
if outEpsg and epsg != outEpsg:
from glass.g.prj.obj import prj_ogrgeom
poly = prj_ogrgeom(polygon, epsg, outEpsg)
return poly
else:
return polygon
def shpext_to_boundary(in_shp, out_srs=None):
"""
Read one feature class extent and create a boundary with that
extent
"""
from glass.g.prop.ext import get_ext
from glass.g.gobj import create_polygon
# Get Extent
ext = get_ext(in_shp)
# Create points of the new boundary based on the extent
boundary_points = [
(ext[0], ext[3]), (ext[1], ext[3]),
(ext[1], ext[2]), (ext[0], ext[2]), (ext[0], ext[3])
]
polygon = create_polygon(boundary_points)
if out_srs:
from glass.g.prop.prj import get_shp_epsg
in_srs = get_shp_epsg(in_shp)
if in_srs != out_srs:
from glass.g.prj.obj import prj_ogrgeom
poly = prj_ogrgeom(polygon, in_srs, out_srs,
api='shply')
return poly
else:
return polygon
else:
return polygon
def getBufferParam(inArea, inAreaSRS, outSRS=4326):
"""
Get Buffer X, Y Center and radius in any SRS (radius in meteres).
Check the type of the 'inArea' Object and return the interest values.
inArea could be a file, dict, list or tuple
"""
import os
from osgeo import ogr
from glass.g.gobj import new_pnt
from glass.g.prj.obj import prj_ogrgeom
TYPE = type(inArea)
if TYPE == str:
# Assuming that inArea is a file
# Check if exists
if os.path.exists(inArea):
if os.path.isfile(inArea):
from glass.g.tbl.filter import geom_by_idx
# Get Geometry object
BUFFER_GEOM = geom_by_idx(inArea, 0)
# To outSRS
if int(inAreaSRS) != outSRS:
BUFFER_GEOM = prj_ogrgeom(
ogr.CreateGeometryFromWkt(BUFFER_GEOM), inAreaSRS,
outSRS).ExportToWkt()
# Get x_center, y_center and radius
xyr = bf_prop(BUFFER_GEOM, outSRS, isFile=None)
x_center, y_center, dist = str(xyr['X']), str(xyr['Y']), str(
xyr['R'])
else:
raise ValueError('The given path exists but it is not a file')
else:
raise ValueError('The given path doesn\'t exist')
elif TYPE == dict:
X = 'x' if 'x' in inArea else 'X' if 'X' in inArea else \
'lng' if 'lng' in inArea else None
Y = 'y' if 'x' in inArea else 'Y' if 'Y' in inArea else \
'lat' if 'lat' in inArea else None
R = 'r' if 'r' in inArea else 'R' if 'R' in inArea else \
'rad' if 'rad' in inArea else 'RADIUS' if 'RADIUS' in inArea \
else 'radius' if 'radius' in inArea else None
if not X or not Y or not R:
raise ValueError(
('The keys used to identify the buffer properties '
'are not valid! '
'Please choose one of the following keys groups: '
'x, y, r; '
'X, Y, R; '
'lat, lng, rad'))
else:
x_center, y_center, dist = (str(inArea[X]), str(inArea[Y]),
str(inArea[R]))
if inAreaSRS != outSRS:
pnt_wgs = prj_ogrgeom(new_pnt(x_center, y_center), inAreaSRS,
outSRS)
x_center, y_center = (pnt_wgs.GetX(), pnt_wgs.GetY())
elif TYPE == list or TYPE == tuple:
x_center, y_center, dist = inArea
if inAreaSRS != outSRS:
pnt_wgs = prj_ogrgeom(new_pnt(x_center, y_center), inAreaSRS,
outSRS)
x_center, y_center = (pnt_wgs.GetX(), pnt_wgs.GetY())
else:
raise ValueError(
('Please give a valid path to a shapefile or a tuple, dict or '
'list with the x, y and radius values'))
return x_center, y_center, dist
def ext_to_rst(topLeft,
btRight,
outRst,
cellsize=None,
epsg=None,
outEpsg=None,
invalidResultAsNull=None,
rstvalue=None):
"""
Extent to Raster
"""
import numpy
from osgeo import gdal
from glass.g.prop import drv_name
left, top = topLeft
right, bottom = btRight
cellsize = 10 if not cellsize else cellsize
if outEpsg and epsg and outEpsg != epsg:
from glass.g.gobj import new_pnt
from glass.g.gobj import create_polygon
from glass.g.prj.obj import prj_ogrgeom
extGeom = prj_ogrgeom(
create_polygon([
new_pnt(left, top),
new_pnt(right, top),
new_pnt(right, bottom),
new_pnt(left, bottom),
new_pnt(left, top)
]), epsg, outEpsg)
epsg = outEpsg
left, right, bottom, top = extGeom.GetEnvelope()
# Get row and cols number
rows = (float(top) - float(bottom)) / cellsize
cols = (float(right) - float(left)) / cellsize
rows = int(rows) if rows == int(rows) else int(rows) + 1
cols = int(cols) if cols == int(cols) else int(cols) + 1
if not invalidResultAsNull:
if not rstvalue:
NEW_RST_ARRAY = numpy.zeros((rows, cols))
else:
NEW_RST_ARRAY = numpy.full((rows, cols), rstvalue)
else:
try:
if not rstvalue:
NEW_RST_ARRAY = numpy.zeros((rows, cols))
else:
NEW_RST_ARRAY = numpy.full((rows, cols), rstvalue)
except:
return None
# Create new Raster
img = gdal.GetDriverByName(drv_name(outRst)).Create(
outRst, cols, rows, 1, gdal.GDT_Byte)
img.SetGeoTransform((left, cellsize, 0, top, 0, -cellsize))
band = img.GetRasterBand(1)
band.WriteArray(NEW_RST_ARRAY)
if epsg:
from osgeo import osr
rstSrs = osr.SpatialReference()
rstSrs.ImportFromEPSG(epsg)
img.SetProjection(rstSrs.ExportToWkt())
band.FlushCache()
return outRst
def photos_location(buffer_shp,
epsg_in,
keyword=None,
epsg_out=4326,
onlySearchAreaContained=True,
keyToUse=None):
"""
Search for data in Flickr and return a array with the same data
buffer_shp cloud be a shapefile with a single buffer feature or a dict
like:
buffer_shp = {
x: x_value,
y: y_value,
r: dist (in meters)
}
or a list or a tuple:
buffer_shp = [x, y, radius]
"""
import pandas
from shapely.geometry import Polygon, Point
from shapely.wkt import loads
from geopandas import GeoDataFrame
from glass.g.gp.prox.bfing.obj import xy_to_buffer
from glass.g.prop.feat.bf import getBufferParam
from glass.g.prj.obj import prj_ogrgeom
x_center, y_center, dist = getBufferParam(buffer_shp, epsg_in, outSRS=4326)
# Retrive data from Flickr
photos = search_photos(lat=y_center,
lng=x_center,
radius=float(dist) / 1000,
keyword=keyword,
apiKey=keyToUse)
try:
if not photos:
# Return noData
return 0
except:
pass
photos['longitude'] = photos['longitude'].astype(float)
photos['latitude'] = photos['latitude'].astype(float)
geoms = [Point(xy) for xy in zip(photos.longitude, photos.latitude)]
gdata = GeoDataFrame(photos, crs='EPSG:4326', geometry=geoms)
if onlySearchAreaContained:
_x_center, _y_center, _dist = getBufferParam(buffer_shp,
epsg_in,
outSRS=3857)
# Check if all retrieve points are within the search area
search_area = xy_to_buffer(float(_x_center), float(_y_center),
float(_dist))
search_area = prj_ogrgeom(search_area, 3857, 4326)
search_area = loads(search_area.ExportToWkt())
gdata["tst_geom"] = gdata["geometry"].intersects(search_area)
gdata = gdata[gdata["tst_geom"] == True]
gdata.reset_index(drop=True, inplace=True)
gdata["fid"] = gdata["id"]
if "url_l" in gdata.columns.values:
gdata["url"] = gdata["url_l"]
else:
gdata["url"] = 'None'
gdata["description"] = gdata["_content"]
# Drop irrelevant fields
cols = list(gdata.columns.values)
delCols = []
for col in cols:
if col != 'geometry' and col != 'description' and \
col != 'fid' and col != 'url' and col != 'datetaken' \
and col != 'dateupload' and col != 'title':
delCols.append(col)
else:
continue
gdata.drop(delCols, axis=1, inplace=True)
if epsg_out != 4326:
gdata = gdata.to_crs('EPSG:{}'.format(str(epsg_out)))
return gdata
def download_by_boundary(input_boundary, folder_out, osm_name, epsg,
GetUrl=True, db_name=None, geomCol=None,
justOneFeature=None):
"""
Download data from OSM using a bounding box
"""
import os
from osgeo import ogr
from glass.pys.web import get_file
from glass.pys.oss import os_name
OS_NAME = os_name()
EXTENTS = []
if db_name and geomCol:
"""
Assuming input_boundary is a PostgreSQL Table
"""
from glass.pys import obj_to_lst
from glass.g.prop.gql import tbl_ext
for t in obj_to_lst(input_boundary):
EXTENTS.append(tbl_ext(db_name, t, geomCol))
else:
if type(input_boundary) == dict:
if 'top' in input_boundary and 'bottom' in input_boundary \
and 'left' in input_boundary and 'right' in input_boundary:
EXTENTS.append([
input_boundary['left'],input_boundary['right'],
input_boundary['bottom'], input_boundary['top']
])
else:
raise ValueError((
'input_boundary is a dict but the keys are not correct. '
'Please use left, right, top and bottom as keys'
))
elif type(input_boundary) == list:
if len(input_boundary) == 4:
EXTENTS.append(input_boundary)
else:
raise ValueError((
'input boundary is a list with more than 4 objects. '
'The list should be like: '
'l = [left, right, bottom, top]'
))
elif type(input_boundary) == ogr.Geometry:
EXTENTS.append(input_boundary.GetEnvelope())
else:
# Assuming input boundary is a file
#Check if file exists
if not os.path.exists(input_boundary):
raise ValueError((
"Sorry, but the file {} does not exist inside the folder {}!"
).format(
os.path.basename(input_boundary),
os.path.dirname(input_boundary)
))
# Check if is a raster
from glass.g.prop import check_isRaster
isRst = check_isRaster(input_boundary)
# Get EPSG
if not epsg:
from glass.g.prop.prj import get_epsg
epsg = get_epsg(input_boundary)
if isRst:
from glass.g.prop.rst import rst_ext
# Get raster extent
EXTENTS.append(rst_ext(input_boundary))
else:
from glass.g.prop import drv_name
# Todo: check if it is shape
# Open Dataset
inSrc = ogr.GetDriverByName(drv_name(
input_boundary)).Open(input_boundary)
lyr = inSrc.GetLayer()
i = 1
for feat in lyr:
geom = feat.GetGeometryRef()
featext = geom.GetEnvelope()
EXTENTS.append(featext)
if justOneFeature:
break
if epsg != 4326:
from glass.g.gobj import new_pnt
from glass.g.prj.obj import prj_ogrgeom
for i in range(len(EXTENTS)):
bottom_left = prj_ogrgeom(new_pnt(
EXTENTS[i][0], EXTENTS[i][2]), epsg, 4326)
top_right = prj_ogrgeom(new_pnt(
EXTENTS[i][1], EXTENTS[i][3]), epsg, 4326)
left , bottom = bottom_left.GetX(), bottom_left.GetY()
right, top = top_right.GetX() , top_right.GetY()
EXTENTS[i] = [left, right, bottom, top]
#url = "https://overpass-api.de/api/map?bbox={}"
url = "https://lz4.overpass-api.de/api/interpreter?bbox={}"
RESULTS = []
for e in range(len(EXTENTS)):
bbox_str = ','.join([str(p) for p in EXTENTS[e]])
if GetUrl:
RESULTS.append(url.format(bbox_str))
continue
if len(EXTENTS) == 1:
outOsm = os.path.join(folder_out, osm_name + '.xml')
else:
outOsm = os.path.join(folder_out, "{}_{}.xml".format(osm_name, str(e)))
osm_file = get_file(
url.format(bbox_str), outOsm,
useWget=None if OS_NAME == 'Windows' else None
)
RESULTS.append(osm_file)
return RESULTS[0] if len(RESULTS) == 1 else RESULTS
def osm_extraction(boundary, osmdata, output, each_feat=None, epsg=None):
"""
Extract OSM Data from a xml file with osmosis
The extraction is done using the extent of a boundary
"""
import os
from glass.pys import execmd
from glass.g.prj.obj import prj_ogrgeom
from glass.g.prop import check_isRaster
# Check if boundary is a file
if os.path.isfile(boundary):
# Check if boundary is a raster
is_rst = check_isRaster(boundary)
if is_rst:
# Get Raster EPSG and Extent
from glass.g.prop.prj import get_rst_epsg
from glass.g.prop.rst import rst_ext
from glass.g.gobj import create_polygon
in_epsg = get_rst_epsg(boundary)
left, right, bottom, top = rst_ext(boundary)
boundaries = [
create_polygon([(left, top), (right, top), (right, bottom),
(left, bottom), (left, top)])
]
else:
# Get Shape EPSG
from glass.g.prop.prj import get_shp_epsg
in_epsg = get_shp_epsg(boundary)
if not each_feat:
# Get Shape Extent
from glass.g.prop.feat import get_ext
from glass.g.gobj import create_polygon
left, right, bottom, top = get_ext(boundary)
boundaries = [
create_polygon([(left, top), (right, top), (right, bottom),
(left, bottom), (left, top)])
]
else:
# Get Extent of each feature
from osgeo import ogr
from glass.g.prop import drv_name
src = ogr.GetDriverByName(drv_name(boundary)).Open(boundary)
lyr = src.GetLayer()
boundaries = [feat.GetGeometryRef() for feat in lyr]
else:
from glass.g.gobj import wkt_to_geom
in_epsg = 4326 if not epsg else epsg
if type(boundary) == str:
# Assuming it is a WKT string
wkt_boundaries = [boundary]
elif type(boundary) == list:
# Assuming it is a List with WKT strings
wkt_boundaries = boundary
else:
raise ValueError('Given boundary has a not valid value')
boundaries = [wkt_to_geom(g) for g in wkt_boundaries]
if None in boundaries:
raise ValueError(
("boundary parameter is a string, but it is not a valid path "
"to a file or a valid WKT string"))
# Get output files
if len(boundaries) == 1:
if os.path.isdir(output):
fn, ff = os.path.splitext(os.path.basename(osmdata))
out_files = [os.path.join(output, "ect_{}.{}".format(fn, ff))]
else:
out_files = [output]
else:
fn, ff = os.path.splitext(os.path.basename(osmdata))
path = output if os.path.isdir(output) else os.path.dirname(output)
out_files = [
os.path.join(path, "ect_{}_{}.{}".format(fn, str(i), ff))
for i in range(len(boundaries))
]
# Extract data using OSMOSIS
cmd = ("osmosis --read-{_f} {dtparse}file={_in} "
"--bounding-box top={t} left={l} bottom={b} right={r} "
"--write-{outext} file={_out}")
for g in range(len(boundaries)):
# Convert boundary to WGS84 -EPSG 4326
geom_wgs = prj_ogrgeom(
boundaries[g], int(in_epsg), 4326,
api='shapely') if int(in_epsg) != 4326 else boundaries[g]
# Get boundary extent
left, right, bottom, top = geom_wgs.GetEnvelope()
# Osmosis shell comand
osmext = os.path.splitext(osmdata)[1]
# Execute command
outcmd = execmd(
cmd.format(
_f='pbf' if osmext == '.pbf' else 'xml',
_in=osmdata,
t=str(top),
l=str(left),
b=str(bottom),
r=str(right),
_out=out_files[g],
outext=os.path.splitext(out_files[g])[1][1:],
dtparse="" if osmext == '.pbf' else "enableDataParsing=no "))
return output
def places_by_query(bfShp, epsgIn, keyword=None, epsgOut=4326,
_limit='100', onlySearchAreaContained=True):
"""
Get absolute location of facebook data using the Facebook API and
Pandas to validate data.
Works only for the 'places' search type
buffer_shp cloud be a shapefile with a single buffer feature
or a dict like:
buffer_shp = {
x: x_value,
y: y_value,
r: dist
}
or a list or a tuple:
buffer_shp = [x, y, r]
"""
import pandas
from geopandas import GeoDataFrame
from shapely.geometry import Polygon, Point
from glass.g.prop.feat.bf import getBufferParam
from glass.g.acq.dsn.fb.search import by_query
search_type = 'place'
x_center, y_center, dist = getBufferParam(bfShp, epsgIn, outSRS=4326)
data = by_query(
search_type, keyword=keyword,
x_center=x_center, y_center=y_center, dist=dist,
limit=_limit, face_fields=[
"location", "name", "category_list", "about",
"checkins", "description", "fan_count"
]
)
try:
if not data:
# Return NoData
return 0
except:
pass
# Sanitize category_list field
data = pandas.concat([
data.drop(["category_list"], axis=1),
data["category_list"].apply(pandas.Series)
], axis=1)
_int_cols = [
c for c in data.columns.values if type(c) == int
]
__int_cols = {
x : "col_{}".format(str(x)) for x in _int_cols
}
data.rename(columns=__int_cols, inplace=True)
data.rename(columns={"id" : "id_1", "name" : "name_1"}, inplace=True)
for k in __int_cols:
data = pandas.concat([
data.drop([__int_cols[k]], axis=1),
data[__int_cols[k]].apply(pandas.Series)
], axis=1)
data.rename(columns={
'id' : 'id_' + str(k+2),
'name' : 'name_' + str(k+2)
}, inplace=True)
if 0 in list(data.columns.values):
data.drop([0], axis=1, inplace=True)
# Pandas dataframe to Geopandas Dataframe
geoms = [Point(xy) for xy in zip(data.longitude, data.latitude)]
data.drop(["latitude", "longitude"], axis=1, inplace=True)
gdata = GeoDataFrame(data, crs='EPSG:4326', geometry=geoms)
if onlySearchAreaContained:
from shapely.wkt import loads
from glass.g.prj.obj import prj_ogrgeom
from glass.g.gp.prox.bfing.obj import xy_to_buffer
# Check if all retrieve points are within the search area
_x_center, _y_center, _dist = getBufferParam(
bfShp, epsgIn, outSRS=3857
)
search_area = xy_to_buffer(
float(_x_center), float(_y_center), float(_dist)
)
search_area = prj_ogrgeom(search_area, 3857, 4326)
search_area = loads(search_area.ExportToWkt())
gdata["tst_geom"] = gdata["geometry"].intersects(search_area)
gdata = gdata[gdata["tst_geom"] == True]
gdata.reset_index(drop=True, inplace=True)
# Sanitize id
gdata["fid"] = gdata["id_1"]
gdata["fb_type"] = search_type
__DROP_COLS = ["id_1", "city", "country", "street", "zip", "located_in"]
DROP_COLS = [c for c in __DROP_COLS if c in gdata.columns.values]
if onlySearchAreaContained:
DROP_COLS.append("tst_geom")
gdata.drop(DROP_COLS, axis=1, inplace=True)
if epsgOut != 4326:
gdata = gdata.to_crs('EPSG:{}'.format(str(epsgOut)))
return gdata
def geotweets_location(inGeom,
epsg_in,
keyword=None,
epsg_out=4326,
onlySearchAreaContained=True,
keyToUse=None):
"""
Search data in Twitter and array with that data
inGeom cloud be a shapefile with a single buffer feature or a dict like:
inGeom = {
x: x_value,
y: y_value,
r: dist (in meters)
}
or a list or a tuple:
inGeom = [x, y, radius]
"""
from shapely.geometry import Polygon, Point
from geopandas import GeoDataFrame
from glass.g.prop.feat.bf import getBufferParam
x_center, y_center, dist = getBufferParam(inGeom, epsg_in, outSRS=4326)
# Extract data from Twitter
data = search_tweets(lat=y_center,
lng=x_center,
radius=float(dist) / 1000,
keyword=keyword,
NR_ITEMS=500,
only_geo=True,
key=keyToUse)
try:
if not data:
return 0
except:
pass
# Pandas to GeoPandas
geoms = [Point(xy) for xy in zip(data.longitude, data.latitude)]
data.drop(["latitude", "longitude"], axis=1, inplace=True)
gdata = GeoDataFrame(data, crs='EPSG:4326', geometry=geoms)
if onlySearchAreaContained:
from shapely.wkt import loads
from glass.g.prj.obj import prj_ogrgeom
from glass.g.gp.prox.bfing.obj import xy_to_buffer
# Check if all retrieve points are within the search area
_x_center, _y_center, _dist = getBufferParam(inGeom,
epsg_in,
outSRS=3857)
search_area = xy_to_buffer(float(_x_center), float(_y_center),
float(_dist))
search_area = prj_ogrgeom(search_area, 3857, 4326)
search_area = loads(search_area.ExportToWkt())
gdata["tst_geom"] = gdata["geometry"].intersects(search_area)
gdata = gdata[gdata["tst_geom"] == True]
gdata.reset_index(drop=True, inplace=True)
gdata.drop("tst_geom", axis=1, inplace=True)
if epsg_out != 4326:
gdata = gdata.to_crs('EPSG:{}'.format(str(epsg_out)))
return gdata