def download_by_boundary(input_boundary, output_osm, epsg, area_threshold=None, GetUrl=True): """ Download data from OSM using a bounding box """ import os from decimal import Decimal from gasp.prop.feat import feat_count from gasp.prop.ext import get_extent from gasp.web import get_file # Check number of features number_feat = feat_count(input_boundary) if number_feat != 1: raise ValueError( ('Your boundary has more than one feature. ' 'Only feature classes with one feature are allowed.')) # Check boundary area if area_threshold: from gasp.cpu.gdl import area_to_dic d_area = area_to_dic(input_boundary) if d_area[0] > area_threshold: raise ValueError('{} has more than than {} square meters'.format( os.path.basename(input_boundary), str(area_threshold))) # Get boundary extent left, right, bottom, top = get_extent(input_boundary, gisApi='ogr') if epsg != 4326: from gasp.to.geom import create_point from gasp.mng.prj import project_geom bottom_left = project_geom(create_point(left, bottom, api='ogr'), epsg, 4326, api='ogr') top_right = project_geom(create_point(right, top, api='ogr'), epsg, 4326, api='ogr') left, bottom = bottom_left.GetX(), bottom_left.GetY() right, top = top_right.GetX(), top_right.GetY() bbox_str = ','.join([str(left), str(bottom), str(right), str(top)]) url = "http://overpass-api.de/api/map?bbox={box}".format(box=bbox_str) if GetUrl: return url osm_file = get_file(url, output_osm) return output_osm
def osmosis_extract(boundary, osmdata, wepsg, output): """ Extract OSM Data from a xml file with osmosis The extraction is done using the extent of a boundary """ import os from osgeo import ogr from gasp import exec_cmd from gasp.prop.ff import drv_name from gasp.mng.prj import project_geom # Assuming that boundary has only one feature # Get Geometry dtSrc = ogr.GetDriverByName(drv_name(boundary)).Open(boundary) lyr = dtSrc.GetLayer() for feat in lyr: geom = feat.GetGeometryRef() break # Convert boundary to WGS84 -EPSG 4326 geom_wgs = project_geom(geom, int(wepsg), 4326, api='ogr') if int(wepsg) != 4326 else geom # Get boundary extent left, right, bottom, top = geom_wgs.GetEnvelope() # Osmosis shell comand osmExt = os.path.splitext(osmdata)[1] osm_f = 'enableDateParsing=no' if osmExt == '.xml' or osmExt == '.osm' else '' cmd = ('osmosis --read-{_f} {p} file={_in} --bounding-box top={t} left={l}' ' bottom={b} right={r} --write-{outext} file={_out}').format( _f='xml' if osmExt == '.xml' or osmExt == '.osm' else 'pbf', p=osm_f, _in=osmdata, t=str(top), l=str(left), b=str(bottom), r=str(right), _out=output, outext=os.path.splitext(output)[1][1:]) # Execute command outcmd = exec_cmd(cmd) return output
def buffer_properties(buffer_shp, epsg_in, isFile=None): """ Return the centroid and radius distance of one buffer geometry Centroid X, Y in the units of the buffer_shp; Radius in meters. Object return will be something like this: o = { 'X': x_value, 'Y': y_value, 'R': radius_value } """ from gasp.prop.feat import get_centroid_boundary if isFile: from gasp.anls.exct import get_geom_by_index BUFFER_GEOM = ogr.CreateGeometryFromWkt( get_geom_by_index(buffer_shp, 0)) else: BUFFER_GEOM = ogr.CreateGeometryFromWkt(buffer_shp) # Get x_center, y_center and dist from polygon geometry # TODO: Besides 4326, we need to include also the others geographic systems if int(epsg_in) == 4326: from gasp.mng.prj import project_geom BUFFER_GEOM_R = project_geom(BUFFER_GEOM, epsg_in, 3857, api='ogr') else: BUFFER_GEOM_R = BUFFER_GEOM dist = get_buffer_radius(BUFFER_GEOM_R.ExportToWkt(), isFile=None) center = get_centroid_boundary(BUFFER_GEOM, isFile=None) return {'X': center.GetX(), 'Y': center.GetY(), 'R': dist}
def download_by_polygeom(inGeom, outOsm, epsg): """ Download data from OSM using extent of a Polygon Object """ from gasp.web import get_file # Transform polygon if necessary if epsg != 4326: from gasp.mng.prj import project_geom inGeom = project_geom(inGeom, epsg, 4326, api='ogr') # Get polygon extent left, right, bottom, top = inGeom.GetEnvelope() bbox_str = "{},{},{},{}".format(str(left), str(bottom), str(right), str(top)) url = "https://overpass-api.de/api/map?bbox={}".format(bbox_str) osm_file = get_file(url, outOsm) return outOsm
def getBufferParam(inArea, inAreaSRS, outSRS=4326): """ Get Buffer X, Y Center and radius in any SRS (radius in meteres). Check the type of the 'inArea' Object and return the interest values. inArea could be a file, dict, list or tuple """ import os from gasp.to.geom import create_point from gasp.mng.prj import project_geom TYPE = type(inArea) if TYPE == str or TYPE == unicode: # Assuming that inArea is a file # Check if exists if os.path.exists(inArea): if os.path.isfile(inArea): from gasp.anls.exct import get_geom_by_index # Get Geometry object BUFFER_GEOM = get_geom_by_index(inArea, 0) # To outSRS if int(inAreaSRS) != outSRS: BUFFER_GEOM = project_geom( ogr.CreateGeometryFromWkt(BUFFER_GEOM), inAreaSRS, outSRS, api='ogr').ExportToWkt() # Get x_center, y_center and radius xyr = buffer_properties(BUFFER_GEOM, outSRS, isFile=None) x_center, y_center, dist = str(xyr['X']), str(xyr['Y']), str( xyr['R']) else: raise ValueError('The given path exists but it is not a file') else: raise ValueError('The given path doesn\'t exist') elif TYPE == dict: X = 'x' if 'x' in inArea else 'X' if 'X' in inArea else \ 'lng' if 'lng' in inArea else None Y = 'y' if 'x' in inArea else 'Y' if 'Y' in inArea else \ 'lat' if 'lat' in inArea else None R = 'r' if 'r' in inArea else 'R' if 'R' in inArea else \ 'rad' if 'rad' in inArea else 'RADIUS' if 'RADIUS' in inArea \ else 'radius' if 'radius' in inArea else None if not X or not Y or not R: raise ValueError( ('The keys used to identify the buffer properties ' 'are not valid! ' 'Please choose one of the following keys groups: ' 'x, y, r; ' 'X, Y, R; ' 'lat, lng, rad')) else: x_center, y_center, dist = (str(inArea[X]), str(inArea[Y]), str(inArea[R])) if inAreaSRS != outSRS: pnt_wgs = project_geom(create_point(x_center, y_center, api='ogr'), inAreaSRS, outSRS, api='ogr') x_center, y_center = (pnt_wgs.GetX(), pnt_wgs.GetY()) elif TYPE == list or TYPE == tuple: x_center, y_center, dist = inArea if inAreaSRS != outSRS: pnt_wgs = project_geom(create_point(x_center, y_center, api='ogr'), inAreaSRS, outSRS, api='ogr') x_center, y_center = (pnt_wgs.GetX(), pnt_wgs.GetY()) else: raise ValueError( ('Please give a valid path to a shapefile or a tuple, dict or ' 'list with the x, y and radius values')) return x_center, y_center, dist
def download_by_boundary(input_boundary, output_osm, epsg, GetUrl=True): """ Download data from OSM using a bounding box """ import os from osgeo import ogr from gasp.web import get_file from gasp.oss import get_fileformat, get_filename if type(input_boundary) == dict: if 'top' in input_boundary and 'bottom' in input_boundary \ and 'left' in input_boundary and 'right' in input_boundary: left, right, bottom, top = (input_boundary['left'], input_boundary['right'], input_boundary['bottom'], input_boundary['top']) else: raise ValueError( ('input_boundary is a dict but the keys are not correct. ' 'Please use left, right, top and bottom as keys')) elif type(input_boundary) == list: if len(input_boundary) == 4: left, right, bottom, top = input_boundary else: raise ValueError( ('input boundary is a list with more than 4 objects. ' 'The list should be like: ' 'l = [left, right, bottom, top]')) elif type(input_boundary) == ogr.Geometry: # Check if we have a polygon geom_name = input_boundary.GetGeometryName() if geom_name == 'POLYGON' or geom_name == 'MULTIPOLYGON': # Get polygon extent left, right, bottom, top = input_boundary.GetEnvelope() else: raise ValueError(('Your boundary is a non POLYGON ogr Geometry ')) else: # Assuming input boundary is a file #Check if file exists if not os.path.exists(input_boundary): raise ValueError( ("Sorry, but the file {} does not exist inside the folder {}!" ).format(os.path.basename(input_boundary), os.path.dirname(input_boundary))) # Check if is a raster from gasp.prop.ff import check_isRaster isRst = check_isRaster(input_boundary) if isRst: from gasp.prop.ext import rst_ext # Get raster extent left, right, bottom, top = rst_ext(input_boundary) else: from gasp.prop.feat import feat_count from gasp.prop.ext import get_extent # Check number of features number_feat = feat_count(input_boundary, gisApi='ogr') if number_feat != 1: raise ValueError( ('Your boundary has more than one feature. ' 'Only feature classes with one feature are allowed.')) # Get boundary extent left, right, bottom, top = get_extent(input_boundary, gisApi='ogr') if epsg != 4326: from gasp.to.geom import create_point from gasp.mng.prj import project_geom bottom_left = project_geom(create_point(left, bottom, api='ogr'), epsg, 4326, api='ogr') top_right = project_geom(create_point(right, top, api='ogr'), epsg, 4326, api='ogr') left, bottom = bottom_left.GetX(), bottom_left.GetY() right, top = top_right.GetX(), top_right.GetY() bbox_str = ','.join([str(left), str(bottom), str(right), str(top)]) url = "https://overpass-api.de/api/map?bbox={box}".format(box=bbox_str) if GetUrl: return url if get_fileformat(output_osm) != '.xml': output_osm = os.path.join(os.path.dirname(output_osm), get_filename(output_osm) + '.xml') osm_file = get_file(url, output_osm) return output_osm
def places_by_query(bfShp, epsgIn, keyword=None, epsgOut=4326, _limit='100', onlySearchAreaContained=True): """ Get absolute location of facebook data using the Facebook API and Pandas to validate data. Works only for the 'places' search type buffer_shp cloud be a shapefile with a single buffer feature or a dict like: buffer_shp = { x: x_value, y: y_value, r: dist } or a list or a tuple: buffer_shp = [x, y, r] """ import pandas from shapely.geometry import Polygon, Point from geopandas import GeoDataFrame from gasp.anls.prox.bf import getBufferParam from gasp.web.dsn.fb.search import by_query search_type = 'place' x_center, y_center, dist = getBufferParam(bfShp, epsgIn, outSRS=4326) data = by_query( search_type, keyword=keyword, x_center=x_center, y_center=y_center, dist=dist, limit=_limit, face_fields=[ "location", "name", "category_list", "about", "checkins", "description", "fan_count" ] ) try: if not data: # Return NoData return 0 except: pass # Sanitize category_list field data = pandas.concat([ data.drop(["category_list"], axis=1), data["category_list"].apply(pandas.Series) ], axis=1) _int_cols = [ c for c in data.columns.values if type(c) == long ] __int_cols = { x : "col_{}".format(str(x)) for x in _int_cols } data.rename(columns=__int_cols, inplace=True) data.rename(columns={"id" : "id_1", "name" : "name_1"}, inplace=True) for k in __int_cols: data = pandas.concat([ data.drop([__int_cols[k]], axis=1), data[__int_cols[k]].apply(pandas.Series) ], axis=1) data.rename(columns={ 'id' : 'id_' + str(k+2), 'name' : 'name_' + str(k+2) }, inplace=True) if long(0) in list(data.columns.values): data.drop([0], axis=1, inplace=True) # Pandas dataframe to Geopandas Dataframe geoms = [Point(xy) for xy in zip(data.longitude, data.latitude)] data.drop(["latitude", "longitude"], axis=1, inplace=True) gdata = GeoDataFrame(data, crs={'init' : 'epsg:4326'}, geometry=geoms) if onlySearchAreaContained: from shapely.wkt import loads from gasp.mng.prj import project_geom from gasp.anls.prox.bf import coord_to_buffer # Check if all retrieve points are within the search area _x_center, _y_center, _dist = getBufferParam( bfShp, epsgIn, outSRS=3857 ) search_area = coord_to_buffer( float(_x_center), float(_y_center), float(_dist) ) search_area = project_geom(search_area, 3857, 4326, api='ogr') search_area = loads(search_area.ExportToWkt()) gdata["tst_geom"] = gdata["geometry"].intersects(search_area) gdata = gdata[gdata["tst_geom"] == True] gdata.reset_index(drop=True, inplace=True) # Sanitize id gdata["fid"] = gdata["id_1"] gdata["fb_type"] = search_type __DROP_COLS = ["id_1", "city", "country", "street", "zip", "located_in"] DROP_COLS = [c for c in __DROP_COLS if c in gdata.columns.values] if onlySearchAreaContained: DROP_COLS.append("tst_geom") gdata.drop(DROP_COLS, axis=1, inplace=True) if epsgOut != 4326: gdata = gdata.to_crs({'init' : 'epsg:{}'.format(str(epsgOut))}) return gdata
def gdl_mean_time_wByPop(unities, unities_groups, population_field, destinations, output, workspace=None, unities_epsg=4326, destinations_epsg=4326): """ Tempo medio ponderado pela populacao residente a infra-estrutura mais proxima # TODO: Migrate to Pandas """ import os from osgeo import ogr from gasp.prop.ff import drv_name from gasp.fm import points_to_list from gasp.mng.feat import feat_to_pnt from gasp.mng.prj import project_geom from gasp.mng.fld import add_fields from gasp.mng.genze import dissolve from gasp.web.glg.direct import get_time_pnt_destinations workspace = workspace if workspace else \ os.path.dirname(output) # Unities to centroid pnt_unities = feat_to_pnt( unities, os.path.join(workspace, 'pnt_' + os.path.basename(unities))) # List destinations lst_destinies = points_to_list(destinations, listVal="dict", inEpsg=destinations_epsg, outEpsg=4326) # Calculate indicator polyUnit = ogr.GetDriverByName(drv_name(unities)).Open(unities, 1) polyLyr = polyUnit.GetLayer() polyLyr = add_fields(polyLyr, {'meantime': ogr.OFTReal}) pntUnit = ogr.GetDriverByName(drv_name(pnt_unities)).Open(pnt_unities, 0) pntLyr = pntUnit.GetLayer() polyFeat = polyLyr.GetNextFeature() distUnities = {} groups = {} for pntFeat in pntLyr: geom = pntFeat.GetGeometryRef() if unities_epsg == 4326: originGeom = geom else: originGeom = project_geom(geom, unities_epsg, 4326, api='ogr') _id, duration, distance = get_time_pnt_destinations( originGeom, lst_destinies) __min = duration['value'] / 60.0 pop = polyFeat.GetField(population_field) group = polyFeat.GetField(unities_groups) distUnities[polyFeat.GetFID()] = (__min, __min * pop) if group not in groups: groups[group] = __min * pop else: groups[group] += __min * pop polyFeat = polyLyr.GetNextFeature() del polyLyr polyUnit.Destroy() polyUnit = ogr.GetDriverByName(drv_name(unities)).Open(unities, 1) polyLyr = polyUnit.GetLayer() for feat in polyLyr: unitId = feat.GetFID() groupId = feat.GetField(unities_groups) indicator = (distUnities[unitId][1] / groups[groupId]) * distUnities[unitId][0] feat.SetField('meantime', indicator) polyLyr.SetFeature(feat) del polyLyr, pntLyr polyUnit.Destroy() pntUnit.Destroy() dissolve(unities, output, unities_groups, statistics={'meantime': 'SUM'}, api='ogr')
def photos_location(buffer_shp, epsg_in, keyword=None, epsg_out=4326, onlySearchAreaContained=True, keyToUse=None): """ Search for data in Flickr and return a array with the same data buffer_shp cloud be a shapefile with a single buffer feature or a dict like: buffer_shp = { x: x_value, y: y_value, r: dist (in meters) } or a list or a tuple: buffer_shp = [x, y, radius] """ import pandas from shapely.geometry import Polygon, Point from shapely.wkt import loads from geopandas import GeoDataFrame from gasp.anls.prox.bf import coord_to_buffer from gasp.anls.prox.bf import getBufferParam from gasp.mng.prj import project_geom x_center, y_center, dist = getBufferParam(buffer_shp, epsg_in, outSRS=4326) # Retrive data from Flickr photos = search_photos(lat=y_center, lng=x_center, radius=float(dist) / 1000, keyword=keyword, apiKey=keyToUse) try: if not photos: # Return noData return 0 except: pass photos['longitude'] = photos['longitude'].astype(float) photos['latitude'] = photos['latitude'].astype(float) geoms = [Point(xy) for xy in zip(photos.longitude, photos.latitude)] gdata = GeoDataFrame(photos, crs={'init': 'epsg:4326'}, geometry=geoms) if onlySearchAreaContained: _x_center, _y_center, _dist = getBufferParam(buffer_shp, epsg_in, outSRS=3857) # Check if all retrieve points are within the search area search_area = coord_to_buffer(float(_x_center), float(_y_center), float(_dist)) search_area = project_geom(search_area, 3857, 4326, api='ogr') search_area = loads(search_area.ExportToWkt()) gdata["tst_geom"] = gdata["geometry"].intersects(search_area) gdata = gdata[gdata["tst_geom"] == True] gdata.reset_index(drop=True, inplace=True) gdata["fid"] = gdata["id"] if "url_l" in gdata.columns.values: gdata["url"] = gdata["url_l"] else: gdata["url"] = 'None' gdata["description"] = gdata["_content"] # Drop irrelevant fields cols = list(gdata.columns.values) delCols = [] for col in cols: if col != 'geometry' and col != 'description' and \ col != 'fid' and col != 'url' and col != 'datetaken' \ and col != 'dateupload' and col != 'title': delCols.append(col) else: continue gdata.drop(delCols, axis=1, inplace=True) if epsg_out != 4326: gdata = gdata.to_crs({'init': 'epsg:{}'.format(str(epsg_out))}) return gdata
def shpext_to_rst(inShp, outRaster, cellsize=None, epsg=None, invalidResultAsNone=None, outEpsg=None): """ Extent to raster if invalidResultAsNone - if for some reason something went wrong, the result of this method will be a None Object if there is an error on the numpy array creation. If False, an error will be raised. """ import os import numpy from osgeo import gdal from gasp.prop.ff import drv_name from gasp.prop.ext import get_extent cellsize = 10 if not cellsize else cellsize # Get extent try: left, right, bottom, top = get_extent(inShp, gisApi='ogr') except: left, right, bottom, top = inShp.GetEnvelope() if outEpsg and epsg and outEpsg != epsg: from gasp.to.geom import create_polygon, create_point from gasp.mng.prj import project_geom extGeom = project_geom(create_polygon([ create_point(left, top, api='ogr'), create_point(right, top, api='ogr'), create_point(right, bottom, api='ogr'), create_point(left, bottom, api='ogr'), create_point(left, top, api='ogr') ]), epsg, outEpsg, api='ogr') left, right, bottom, top = extGeom.GetEnvelope() # Get row and cols number rows = (float(top) - float(bottom)) / cellsize cols = (float(right) - float(left)) / cellsize rows = int(rows) if rows == int(rows) else int(rows) + 1 cols = int(cols) if cols == int(cols) else int(cols) + 1 if not invalidResultAsNone: NEW_RASTER_ARRAY = numpy.zeros((rows, cols)) else: try: NEW_RASTER_ARRAY = numpy.zeros((rows, cols)) except: return None # Create new raster img = gdal.GetDriverByName(drv_name(outRaster)).Create( outRaster, cols, rows, 1, #gdal.GDT_Int16 gdal.GDT_Byte) img.SetGeoTransform((left, cellsize, 0, top, 0, -cellsize)) band = img.GetRasterBand(1) band.WriteArray(NEW_RASTER_ARRAY) if epsg: from osgeo import osr rstSrs = osr.SpatialReference() rstSrs.ImportFromEPSG(epsg) img.SetProjection(rstSrs.ExportToWkt()) band.FlushCache() return outRaster
def geotweets_location(inGeom, epsg_in, keyword=None, epsg_out=4326, onlySearchAreaContained=True, keyToUse=None): """ Search data in Twitter and array with that data inGeom cloud be a shapefile with a single buffer feature or a dict like: inGeom = { x: x_value, y: y_value, r: dist (in meters) } or a list or a tuple: inGeom = [x, y, radius] """ from shapely.geometry import Polygon, Point from geopandas import GeoDataFrame from gasp.anls.prox.bf import getBufferParam x_center, y_center, dist = getBufferParam(inGeom, epsg_in, outSRS=4326) # Extract data from Twitter data = search_tweets(lat=y_center, lng=x_center, radius=float(dist) / 1000, keyword=keyword, NR_ITEMS=500, only_geo=True, key=keyToUse) try: if not data: return 0 except: pass # Pandas to GeoPandas geoms = [Point(xy) for xy in zip(data.longitude, data.latitude)] data.drop(["latitude", "longitude"], axis=1, inplace=True) gdata = GeoDataFrame(data, crs={'init': 'epsg:4326'}, geometry=geoms) if onlySearchAreaContained: from shapely.wkt import loads from gasp.mng.prj import project_geom from gasp.anls.prox.bf import coord_to_buffer # Check if all retrieve points are within the search area _x_center, _y_center, _dist = getBufferParam(inGeom, epsg_in, outSRS=3857) search_area = coord_to_buffer(float(_x_center), float(_y_center), float(_dist)) search_area = project_geom(search_area, 3857, 4326, api='ogr') search_area = loads(search_area.ExportToWkt()) gdata["tst_geom"] = gdata["geometry"].intersects(search_area) gdata = gdata[gdata["tst_geom"] == True] gdata.reset_index(drop=True, inplace=True) gdata.drop("tst_geom", axis=1, inplace=True) if epsg_out != 4326: gdata = gdata.to_crs({'init': 'epsg:{}'.format(str(epsg_out))}) return gdata
def geovideos_to_array(search_words, epsg_out=4326, spatial_filter=None, epsg_filter=4326): """ Locate videos on youtube and save these locations to a vectorial file """ import os from gasp.oss.ops import del_files_by_name from gasp.to.geom import create_point from gasp.mng.prj import project_geom from gasp.anls.tplgy import point_in_polygon from gasp.anls.prox.bf import draw_buffer videos = get_video_details_by_keyword(search_words) videos_with_location = [] for video in videos: if video['y'] and video['x']: videos_with_location.append(video) if not len(videos_with_location): # Return nodata return 0 if spatial_filter: from gasp.anls.prox.bf import getBufferParam x_center, y_center, dist = getBufferParam( spatial_filter, epsg_filter) bufferCenter = project_geom( create_point(x_center, y_center, api='ogr'), 4326, 3857, api='ogr' ) bufferGeom = draw_buffer(bufferCenter, dist) filterData = [] for instance in videos_with_location: # Create point WGS_POINT = create_point( float(instance['x']), float(instance['y']), api='ogr' ) point = project_geom(WGS_POINT, 4326, 3857, api='ogr') isPointInPolygon = point_in_polygon(point, bufferGeom) if isPointInPolygon: if epsg_out != 4326: trans_point = project_geom(WGS_POINT, 4326, epsg_out, api='ogr') instance['x'] = trans_point.GetX() instance['y'] = trans_point.GetY() filterData.append(instance) return filterData