def get_matrix(origins, key):
subOrigins = split_df(origins, 10)
for df in subOrigins:
for __df in lst_destinos:
matrix = dist_matrix(str(df.geom.str.cat(sep="|")),
str(__df.geom.str.cat(sep="|")),
df.shape[0],
__df.shape[0],
transport_mode=transMode,
useKey=str(key))
matrix = pandas.DataFrame(matrix)
matrix = pandas.concat([
matrix.drop(["elements"], axis=1),
matrix["elements"].apply(pandas.Series)
],
axis=1)
originsFID = df.old_fid.tolist()
destinaFID = __df.old_fid.tolist()
mm = []
for i in range(len(originsFID)):
for e in range(len(destinaFID)):
ll = [originsFID[i], destinaFID[e], matrix.iloc[i, e]]
mm.append(ll)
Fmatrix = pandas.DataFrame(
mm, columns=["fid_origin", "fid_destin", "cost"])
results.append(Fmatrix)
time.sleep(5)
def elevation_to_pntshp(pnt_shp, epsg, fld_name='ELEVATION'):
"""
Add an elevation attribute to a point feature class
"""
from gasp.fm import tbl_to_obj
from gasp.prop.feat import get_geom_type
from gasp.mng.prj import project
from gasp.mng.split import split_df
from gasp.to.obj import df_to_dict
from gasp.to.shp import df_to_shp
# Check Geometries type - shapes should be of type point
geomt = get_geom_type(pnt_shp, name=True, gisApi='ogr')
if geomt != 'POINT' and geomt != 'MULTIPOINT':
raise ValueError('All input geometry must be of type point')
src = tbl_to_obj(pnt_shp)
if epsg != 4326:
src = project(src, None, 4326, gisApi='pandas')
# Get str with coords
src["coords"] = src["geometry"].y.astype(str) + "," + \
src["geometry"].x.astype(str)
# Split dataframe
dfs = split_df(src, 250)
for df in dfs:
coord_str = str(df.coords.str.cat(sep="|"))
elvd = pnts_elev(coord_str)
data = elvd
return data
def matrix_od(originsShp, destinationShp, originsEpsg, destinationEpsg,
resultShp, modeTrans="driving"):
"""
Use Pandas to Retrieve data from MapBox Matrix OD Service
"""
import time
from threading import Thread
from gasp.web.mapbx import get_keys, matrix
from gasp.fm import tbl_to_obj
from gasp.mng.split import split_df, split_df_inN
from gasp.mng.fld.df import listval_to_newcols
from gasp.fm.geom import pointxy_to_cols
from gasp.mng.prj import project
from gasp.mng.gen import merge_df
from gasp.prop.feat import get_geom_type
from gasp.to.shp import df_to_shp
# Data to GeoDataFrame
origens = tbl_to_obj( originsShp)
destinos = tbl_to_obj(destinationShp)
# Check if SHPs are points
inGeomType = get_geom_type(origens, geomCol="geometry", gisApi='pandas')
if inGeomType != 'Point' and inGeomType != 'MultiPoint':
raise ValueError('The input geometry must be of type point')
inGeomType = get_geom_type(destinos, geomCol="geometry", gisApi='pandas')
if inGeomType != 'Point' and inGeomType != 'MultiPoint':
raise ValueError('The input geometry must be of type point')
# Re-Project data to WGS
if originsEpsg != 4326:
origens = project(origens, None, 4326, gisApi='pandas')
if destinationEpsg != 4326:
destinos = project(destinos, None, 4326, gisApi='pandas')
origens = pointxy_to_cols(
origens, geomCol="geometry",
colX="longitude", colY="latitude"
); destinos = pointxy_to_cols(
destinos, geomCol="geometry",
colX="longitude", colY="latitude"
)
# Prepare coordinates Str
origens["location"] = origens.longitude.astype(str) \
+ "," + origens.latitude.astype(str)
destinos["location"] = destinos.longitude.astype(str) \
+ "," + destinos.latitude.astype(str)
# Split destinations DataFrame into Dafaframes with
# 24 rows
lst_destinos = split_df(destinos, 24)
# Get Keys to use
KEYS = get_keys()
# Split origins by key
origensByKey = split_df_inN(origens, KEYS.shape[0])
lst_keys= KEYS["key"].tolist()
# Produce matrix
results = []
def get_matrix(origins, key):
def def_apply(row):
rowResults = []
for df in lst_destinos:
strDest = df.location.str.cat(sep=";")
strLocations = row["location"] + ";" + strDest
dados = matrix(
strLocations, idxSources="0",
idxDestinations=";".join([str(i) for i in range(1, df.shape[0] + 1)]),
useKey=key, modeTransportation=modeTrans
)
time.sleep(5)
rowResults += dados["durations"][0]
row["od_matrix"] = rowResults
return row
newOrigins = origins.apply(
lambda x: def_apply(x), axis=1
)
results.append(newOrigins)
# Create threads
thrds = []
i = 1
for df in origensByKey:
thrds.append(Thread(
name="tk{}".format(str(i)), target=get_matrix,
args=(df, lst_keys[i - 1])
))
i += 1
# Start all threads
for thr in thrds:
thr.start()
# Wait for all threads to finish
for thr in thrds:
thr.join()
# Join all dataframes
RESULT = merge_df(results, ignIndex=False)
RESULT = listval_to_newcols(RESULT, "od_matrix")
RESULT.rename(
columns={
c: "dest_{}".format(c)
for c in RESULT.columns.values if type(c) == int or type(c) == long
}, inplace=True
)
if originsEpsg != 4326:
RESULT = project(RESULT, None, originsEpsg, gisApi='pandas')
return df_to_shp(RESULT, resultShp)
return results
def dist_matrix_by_shp(oShp, dShp, oEpsg, dEpsg, result, transMode=None):
"""
Create distance matrix using shapes and Google Maps API
- Uses my first API_KEY
"""
import time
import pandas
from gasp.fm import tbl_to_obj
from gasp.mng.split import split_df
from gasp.mng.prj import project
from gasp.mng.fld.df import listval_to_newcols
from gasp.prop.feat import get_geom_type
from gasp.mng.gen import merge_df
from gasp.web.glg.distmx import dist_matrix
from gasp.to import obj_to_tbl
from gasp.to.obj import df_to_list
from gasp.oss import get_filename
# Origins and Destionations to GeoDataframe
originsDf = tbl_to_obj(oShp)
destnatDf = tbl_to_obj(dShp)
# Check Geometries type - shapes should be of type point
originsGeom = get_geom_type(originsDf, gisApi='pandas')
destGeom = get_geom_type(destnatDf, gisApi='pandas')
if (originsGeom != 'Point' and originsGeom != 'MultiPoint') or \
(destGeom != 'Point' and destGeom != 'MultiPoint'):
raise ValueError('All input geometries must be of type point')
# Re-project GeoDataframes if needed
originsDf = originsDf if oEpsg == 4326 else \
project(originsDf, None, 4326, gisApi='pandas')
destnatDf = destnatDf if dEpsg == 4326 else \
project(destnatDf, None, 4326, gisApi='pandas')
# Geom to Field as str
originsDf["geom"] = originsDf["geometry"].y.astype(str) + "," + \
originsDf["geometry"].x.astype(str)
destnatDf["geom"] = destnatDf["geometry"].y.astype(str) + "," + \
destnatDf["geometry"].x.astype(str)
originsDf["old_fid"] = originsDf.index
destnatDf["old_fid"] = destnatDf.index
# Split Destinations
lstOrigins = split_df(originsDf, 95)
for odf in lstOrigins:
odf.reset_index(inplace=True)
lstDestinations = df_to_list(destnatDf)
RESULTS = []
for destino in lstDestinations:
for oDf in lstOrigins:
matrix = dist_matrix(
str(oDf.geom.str.cat(sep="|")),
str(destino["geom"]),
oDf.shape[0],
1,
transport_mode=transMode,
useKey='AIzaSyAmyPmqtxD20urqtpCpn4ER74a6J4N403k')
matrix = pandas.DataFrame(matrix)
matrix = listval_to_newcols(matrix, "elements")
matrix = matrix.merge(oDf,
how='inner',
left_index=True,
right_index=True)
matrix.rename(columns={
'old_fid': "fid_origin",
0: "cost"
},
inplace=True)
matrix["fid_destin"] = destino['old_fid']
RESULTS.append(matrix)
time.sleep(5)
# Join all dataframes
RESULT = merge_df(RESULTS, ignIndex=False)
RESULT = sanitizeDataCols(RESULT, "cost")
RESULT.drop([
x
for x in originsDf.columns.values if x != "geometry" and x != "old_fid"
],
axis=1,
inplace=True)
RESULT.rename(columns={"geometry": "origin_geom"}, inplace=True)
RESULT = RESULT.merge(destnatDf,
how='inner',
left_on=["fid_destin"],
right_on=["old_fid"])
RESULT.drop([x for x in destnatDf.columns.values if x != "geometry"],
axis=1,
inplace=True)
RESULT.rename(columns={"geometry": "destin_geom"}, inplace=True)
RESULT["origin_geom"] = RESULT.origin_geom.astype(str)
RESULT["destin_geom"] = RESULT.destin_geom.astype(str)
obj_to_tbl(RESULT, result, sheetsName=get_filename(result))
return result
def dist_matrix_using_shp(originsShp,
destinationsShp,
originsEpsg,
destinationsEpsg,
outTable,
transMode=None):
"""
Create a distance matrix using shapes and Google Maps API
"""
import time
from threading import Thread
from gasp.mng.split import split_df, split_df_inN
from gasp.mng.prj import project
from gasp.prop.feat import get_geom_type
from gasp.mng.gen import merge_df
from gasp.fm import tbl_to_obj
from gasp.to import obj_to_tbl
from gasp.web.glg import get_keys
from gasp.web.glg.distmx import dist_matrix
# Origins and Destionations to GeoDataframe
originsDf = tbl_to_obj(originsShp)
destnatDf = tbl_to_obj(destinationsShp)
# Check Geometries type - shapes should be of type point
originsGeom = get_geom_type(originsDf, gisApi='pandas')
destGeom = get_geom_type(destnatDf, gisApi='pandas')
if (originsGeom != 'Point' and originsGeom != 'MultiPoint') or \
(destGeom != 'Point' and destGeom != 'MultiPoint'):
raise ValueError('All input geometries must be of type point')
# Re-project GeoDataframes if needed
originsDf = originsDf if originsEpsg == 4326 else \
project(originsDf, None, 4326, gisApi='pandas')
destnatDf = destnatDf if destinationsEpsg == 4326 else \
project(destnatDf, None, 4326, gisAPi='pandas')
# Geom to Field as str
originsDf["geom"] = originsDf["geometry"].y.astype(str) + "," + \
originsDf["geometry"].x.astype(str)
destnatDf["geom"] = destnatDf["geometry"].y.astype(str) + "," + \
destnatDf["geometry"].x.astype(str)
originsDf["old_fid"] = originsDf.index
destnatDf["old_fid"] = destnatDf.index
# Split destinations DataFrame into Dafaframes with
lst_destinos = split_df(destnatDf, 10)
# Get Keys
KEYS = get_keys()
lst_keys = KEYS["key"].tolist()
origensByKey = split_df_inN(originsDf, KEYS.shape[0])
if len(origensByKey) == len(lst_keys) + 1:
origensByKey[-2] = origensByKey[-2].append(origensByKey[-1])
del origensByKey[-1]
# Produce matrix for each origins in origensByKey
results = []
def get_matrix(origins, key):
subOrigins = split_df(origins, 10)
for df in subOrigins:
for __df in lst_destinos:
matrix = dist_matrix(str(df.geom.str.cat(sep="|")),
str(__df.geom.str.cat(sep="|")),
df.shape[0],
__df.shape[0],
transport_mode=transMode,
useKey=str(key))
matrix = pandas.DataFrame(matrix)
matrix = pandas.concat([
matrix.drop(["elements"], axis=1),
matrix["elements"].apply(pandas.Series)
],
axis=1)
originsFID = df.old_fid.tolist()
destinaFID = __df.old_fid.tolist()
mm = []
for i in range(len(originsFID)):
for e in range(len(destinaFID)):
ll = [originsFID[i], destinaFID[e], matrix.iloc[i, e]]
mm.append(ll)
Fmatrix = pandas.DataFrame(
mm, columns=["fid_origin", "fid_destin", "cost"])
results.append(Fmatrix)
time.sleep(5)
# Create threads
thrds = []
i = 1
for df in origensByKey:
thrds.append(
Thread(name="tk{}".format(str(i)),
target=get_matrix,
args=(df, lst_keys[i - 1])))
i += 1
# Start all threads
for thr in thrds:
thr.start()
# Wait for all threads to finish
for thr in thrds:
thr.join()
# Join all dataframes
RESULT = merge_df(results, ignIndex=False)
RESULT = sanitizeDataCols(RESULT, "cost")
RESULT = RESULT.merge(originsDf,
how='inner',
left_on=["fid_origin"],
right_on=["old_fid"])
RESULT.drop([x for x in originsDf.columns.values if x != "geometry"],
axis=1,
inplace=True)
RESULT.rename(columns={"geometry": "origin_geom"}, inplace=True)
RESULT = RESULT.merge(destnatDf,
how='inner',
left_on=["fid_destin"],
right_on=["old_fid"])
RESULT.drop([x for x in destnatDf.columns.values if x != "geometry"],
axis=1,
inplace=True)
RESULT.rename(columns={"geometry": "destin_geom"}, inplace=True)
RESULT["origin_geom"] = RESULT.origin_geom.astype(str)
RESULT["destin_geom"] = RESULT.destin_geom.astype(str)
return obj_to_tbl(RESULT, outTable)
def get_points_elv(pntShp, output, epsg, elevationColumn="openelv"):
"""
Extract elevation for several points
"""
import pandas
from threading import Thread
from gasp.fm import tbl_to_obj
from gasp.mng.split import split_df
from gasp.mng.prj import project
from gasp.fm.geom import pointxy_to_cols
from gasp.mng.gen import merge_df
from gasp.prop.feat import df_geom_type
from gasp.to.obj import df_to_list
from gasp.to.shp import df_to_shp
# SHP TO DF
df = tbl_to_obj(pntShp)
# Check Geometry - shape should be of type point
dfGeom = get_geom_type(df, geomCol="geometry", gisApi='pandas')
if dfGeom != 'Point' and dfGeom != 'MultiPoint':
raise ValueError('Geometries must be of type point')
# Re-project GeoDataframes if needed
if epsg != 4326:
df = project(df, None, 4326, gisApi='pandas')
df = pointxy_to_cols(df,
geomCol="geometry",
colX="longitude",
colY="latitude")
df2 = df.drop(
[c for c in df.columns.values if c != "longitude" and c != "latitude"],
axis=1,
inplace=False)
dfs = split_df(df2, 200)
RESULTS = []
# Go to the net and extract elevation
def extraction(pntDf):
locations = df_to_list(pntDf)
__result = locations_elev({"locations": locations})
RESULTS.append(pandas.DataFrame(__result["results"]))
thrds = []
for i in range(len(dfs)):
thrds.append(
Thread(name="elvt{}".format(str(i)),
target=extraction,
args=(dfs[i], )))
for t in thrds:
t.start()
for t in thrds:
t.join()
finalDf = merge_df(RESULTS, ignIndex=True)
finalDf.rename(columns={"elevation": elevationColumn}, inplace=True)
# Join with Original Shape
df["long_join"] = df.longitude.round(6) * 1000000
df["long_join"] = df.long_join.astype(int)
df["lat_join"] = df.latitude.round(6) * 1000000
df["lat_join"] = df.lat_join.astype(int)
finalDf["jlat"] = finalDf.latitude.round(6) * 1000000
finalDf["jlat"] = finalDf.jlat.astype(int)
finalDf["jlng"] = finalDf.longitude.round(6) * 1000000
finalDf["jlng"] = finalDf.jlng.astype(int)
newDf = df.merge(finalDf,
how="outer",
left_on=["long_join", "lat_join"],
right_on=["jlng", "jlat"])
if epsg != 4326:
newDf = project(newDf, None, epsg, gisApi='pandas')
newDf.drop([
"longitude_x", "longitude_y", "latitude_x", "latitude_y", "long_join",
"lat_join", "jlng", "jlat"
],
axis=1,
inplace=True)
return df_to_shp(newDf, output)