def _get_sdf(self, path_directory):
"""
Load the photos into a spatial data frame.
:param path_directory: Path to directory containing photos.
:return: SpatialDataFrame
"""
# load up the photos as a list of dictionaries
photo_list = [
Photo(file).dictionary for file in os.listdir(path_directory)
if file.lower().endswith('.jpg')
]
# convert this list of dictionaries into a SpatialDataFrame
photo_sdf = SpatialDataFrame(photo_list)
# set the geometry field property
photo_sdf.set_geometry('SHAPE',
inplace=True,
sr=SpatialReference(wkid=4326))
# reindex all the data and return the result
return photo_sdf.reset_index(drop=True)
def spatial_join(df1,
df2,
left_tag="_left",
right_tag="_right",
keep_all=True):
"""
Joins two spatailly enabled dataframes based on spatial location based
on if the two geometries are intersected.
Parameters:
:df1: left spatial dataframe
:df2: right spatial dataframe
:left_tag: if the same column is in the left and right dataframe,
this will append that string value to the field
:right_tag: if the same column is in the left and right dataframe,
this will append that string value to the field
:keep_all: if set to true all df1 will be kept regardless of spatial
matches
:output:
Spatial Dataframe
"""
import numpy as np
import pandas as pd
from arcgis.features import SpatialDataFrame
if not isinstance(df1, SpatialDataFrame):
raise ValueError("df1 must be a spatial dataframe")
if not isinstance(df2, SpatialDataFrame):
raise ValueError("df2 must be a spatial dataframe")
right_index = df2.sindex
join_idx = []
if not df1.geometry is None:
geom_field = df1.geometry.name
else:
raise ValueError("df1 is missing a geometry column")
if not df2.geometry is None:
geom_field2 = df2.geometry.name
else:
raise ValueError("df2 is missing a geometry column")
for idx, row in df1.iterrows():
geom = row[geom_field]
if isinstance(geom.extent, tuple):
ext = (geom.extent[0], geom.extent[1], geom.extent[2],
geom.extent[3])
else:
ext = (geom.extent.XMin, geom.exten.YMin, geom.extent.XMax,
geom.extent.YMax)
select_idx = right_index.intersect(ext)
if len(select_idx) > 0:
sub = df2.loc[select_idx]
res = sub[sub.disjoint(geom) == False]
if len(res) > 0:
for idx2, row2 in res.iterrows():
join_idx.append([idx, idx2])
del idx2, row2
elif len(res) == 0 and keep_all:
join_idx.append([idx, None])
del sub, res
elif len(select_idx) == 0 and \
keep_all:
join_idx.append([idx, None])
del geom
del ext
del select_idx
del idx
join_field_names = ["TARGET_OID", "JOIN_OID"]
df2 = df2.copy()
del df2[df2.geometry.name]
join_df = pd.DataFrame(data=join_idx, columns=join_field_names)
join_df = join_df.merge(df1,
left_on=join_field_names[0],
right_index=True,
how='left',
suffixes=(left_tag, right_tag))
join_df = join_df.merge(df2,
left_on=join_field_names[1],
right_index=True,
how='left',
suffixes=(left_tag, right_tag),
copy=True)
join_df = SpatialDataFrame(join_df)
join_df.geometry = join_df[df1.geometry.name]
del join_idx
join_df.reset_index(drop=True, inplace=True)
return join_df
def completeness(out_sdf, df_list, osm_sdf):
print('Running Completeness')
for idx, row in enumerate(out_sdf.iterrows()):
before_val = None
geom = Geometry(row[1].SHAPE)
# Unpack Geom Extent as OSM Expects
bbox = (geom.extent[1], geom.extent[0], geom.extent[3], geom.extent[2])
# Fetch OSM SpatialDataFrame
osm_sdf = gen_osm_sdf('line', bbox, osm_tag='highway')
data_sdf = df_list[idx]
if len(data_sdf) == 0:
before_val = 0
else:
sq = data_sdf[data_sdf.geometry.notnull()].geometry.disjoint(
geom) == False
df_before = data_sdf[sq].copy()
geoms_before = df_before.clip(geom.extent)
geoms_before_sdf = SpatialDataFrame(geometry=geoms_before)
q_before = geoms_before_sdf['SHAPE'] == {"paths": []}
geoms_before_sdf = geoms_before_sdf[~q_before].copy()
geoms_before_sdf.reset_index(inplace=True, drop=True)
geometry_type = osm_sdf.geometry_type
sq = osm_sdf[osm_sdf.geometry.notnull()].geometry.disjoint(
geom) == False
df_after = osm_sdf[sq].copy()
geoms_after = df_after.clip(geom.extent)
geoms_after_sdf = SpatialDataFrame(geometry=geoms_after)
#geoms_after_sdf = SpatialDataFrame({'Pass': '******'}, geometry=geoms_after, index=[0])
q_after = geoms_after_sdf['SHAPE'] == {"paths": []}
geoms_after_sdf = geoms_after_sdf[~q_after].copy()
geoms_after_sdf.reset_index(inplace=True, drop=True)
# This Need Work
if geometry_type == "Polygon":
if before_val == None:
before_val = geoms_before_sdf.geometry.project_as(
4326).get_area('GEODESIC', 'SQUAREKILOMETERS').sum()
after_val = geoms_after_sdf.geometry.project_as(4326).get_area(
'GEODESIC', 'SQUAREKILOMETERS').sum()
if after_val > 0:
score = get_cp_score(ratio=before_val / after_val,
baseVal=before_val,
inputVal=after_val)
else:
score = get_cp_score(0, before_val, after_val)
out_sdf.set_value(idx,
field_schema.get('cmpl')[0],
round(before_val, 1))
out_sdf.set_value(idx,
field_schema.get('cmpl')[1], round(after_val, 1))
out_sdf.set_value(idx,
field_schema.get('cmpl')[3],
round(before_val - after_val, 1))
out_sdf.set_value(idx, field_schema.get('cmpl')[2], score)
elif geometry_type == "Polyline":
if before_val == None:
geom = geoms_before_sdf.geometry
geom_projected = geoms_before_sdf.geometry.project_as(3857)
before_val = int(sum(geom_projected.length.tolist()))
geom_projected = geoms_before_sdf.geometry.project_as(3857)
after_val = int(sum(geom_projected.length.tolist()))
if after_val > 0:
score = get_cp_score(ratio=before_val / after_val,
baseVal=before_val,
inputVal=after_val)
else:
score = get_cp_score(0, before_val, after_val)
out_sdf.set_value(idx,
field_schema.get('cmpl')[0],
round(before_val, 1))
out_sdf.set_value(idx,
field_schema.get('cmpl')[1], round(after_val, 1))
out_sdf.set_value(idx,
field_schema.get('cmpl')[3],
round(before_val - after_val, 1))
out_sdf.set_value(idx, field_schema.get('cmpl')[2], score)
else:
if before_val == None:
before_count = len(geoms_before_sdf)
else:
before_count = 0
after_count = len(geoms_after_sdf)
if after_count > 0:
score = get_cp_score(ratio=before_count / after_count,
baseVal=before_count,
inputVal=after_count)
else:
score = get_cp_score(ratio=0,
baseVal=before_count,
inputVal=after_count)
out_sdf.set_value(idx, field_schema.get('cmpl')[0], before_count)
out_sdf.set_value(idx, field_schema.get('cmpl')[1], after_count)
out_sdf.set_value(idx,
field_schema.get('cmpl')[3],
before_count - after_count)
out_sdf.set_value(idx, field_schema.get('cmpl')[2], score)
del sq
del df_after
del geom
if before_val != None:
print(before_val)
# del df_before
return out_sdf
def from_featureclass(filename, **kwargs):
"""
Returns a GeoDataFrame from a feature class.
Inputs:
filename: full path to the feature class
Optional Parameters:
sql_clause: sql clause to parse data down
where_clause: where statement
sr: spatial reference object
fields: list of fields to extract from the table
"""
from .. import SpatialDataFrame
from arcgis.geometry import _types
if HASARCPY:
sql_clause = kwargs.pop('sql_clause', (None,None))
where_clause = kwargs.pop('where_clause', None)
sr = kwargs.pop('sr', arcpy.Describe(filename).spatialReference or arcpy.SpatialReference(4326))
fields = kwargs.pop('fields', None)
desc = arcpy.Describe(filename)
if not fields:
fields = [field.name for field in arcpy.ListFields(filename) \
if field.type not in ['Geometry']]
if hasattr(desc, 'areaFieldName'):
afn = desc.areaFieldName
if afn in fields:
fields.remove(afn)
if hasattr(desc, 'lengthFieldName'):
lfn = desc.lengthFieldName
if lfn in fields:
fields.remove(lfn)
geom_fields = fields + ['SHAPE@']
flds = fields + ['SHAPE']
vals = []
geoms = []
geom_idx = flds.index('SHAPE')
shape_type = desc.shapeType
default_polygon = _types.Geometry(arcpy.Polygon(arcpy.Array([arcpy.Point(0,0)]* 3)))
default_polyline = _types.Geometry(arcpy.Polyline(arcpy.Array([arcpy.Point(0,0)]* 2)))
default_point = _types.Geometry(arcpy.PointGeometry(arcpy.Point()))
default_multipoint = _types.Geometry(arcpy.Multipoint(arcpy.Array([arcpy.Point()])))
with arcpy.da.SearchCursor(filename,
field_names=geom_fields,
where_clause=where_clause,
sql_clause=sql_clause,
spatial_reference=sr) as rows:
for row in rows:
row = list(row)
# Prevent curves/arcs
if row[geom_idx] is None:
row.pop(geom_idx)
g = {}
elif row[geom_idx].type in ['polyline', 'polygon']:
g = _types.Geometry(row.pop(geom_idx).generalize(0))
else:
g = _types.Geometry(row.pop(geom_idx))
if g == {}:
if shape_type.lower() == 'point':
g = default_point
elif shape_type.lower() == 'polygon':
g = default_polygon
elif shape_type.lower() == 'polyline':
g = default_point
elif shape_type.lower() == 'multipoint':
g = default_multipoint
geoms.append(g)
vals.append(row)
del row
del rows
df = pd.DataFrame(data=vals, columns=fields)
sdf = SpatialDataFrame(data=df, geometry=geoms)
sdf.reset_index(drop=True, inplace=True)
del df
if sdf.sr is None:
if sr is not None:
sdf.sr = sr
else:
sdf.sr = sdf.geometry[sdf.geometry.first_valid_index()].spatialReference
return sdf
elif HASARCPY == False and \
HASPYSHP == True and\
filename.lower().find('.shp') > -1:
geoms = []
records = []
reader = shapefile.Reader(filename)
fields = [field[0] for field in reader.fields if field[0] != 'DeletionFlag']
for r in reader.shapeRecords():
atr = dict(zip(fields, r.record))
g = r.shape.__geo_interface__
g = _geojson_to_esrijson(g)
geom = _types.Geometry(g)
atr['SHAPE'] = geom
records.append(atr)
del atr
del r, g
del geom
sdf = SpatialDataFrame(records)
sdf.set_geometry(col='SHAPE')
sdf.reset_index(inplace=True)
return sdf
elif HASARCPY == False and \
HASPYSHP == False and \
HASFIONA == True and \
(filename.lower().find('.shp') > -1 or \
os.path.dirname(filename).lower().find('.gdb') > -1):
is_gdb = os.path.dirname(filename).lower().find('.gdb') > -1
if is_gdb:
with fiona.drivers():
from arcgis.geometry import _types
fp = os.path.dirname(filename)
fn = os.path.basename(filename)
geoms = []
atts = []
with fiona.open(path=fp, layer=fn) as source:
meta = source.meta
cols = list(source.schema['properties'].keys())
for idx, row in source.items():
geoms.append(_types.Geometry(row['geometry']))
atts.append(list(row['properties'].values()))
del idx, row
df = pd.DataFrame(data=atts, columns=cols)
return SpatialDataFrame(data=df, geometry=geoms)
else:
with fiona.drivers():
from arcgis.geometry import _types
geoms = []
atts = []
with fiona.open(path=filename) as source:
meta = source.meta
cols = list(source.schema['properties'].keys())
for idx, row in source.items():
geoms.append(_types.Geometry(row['geometry']))
atts.append(list(row['properties'].values()))
del idx, row
df = pd.DataFrame(data=atts, columns=cols)
return SpatialDataFrame(data=df, geometry=geoms)
return
def completeness(gis, df_after, df_before, output_features, grid_filter, geom):
""" main driver of program """
try:
out_fl = FeatureLayer(gis=gis, url=output_features)
out_sdf = out_fl.query(geometry_filter=grid_filter,return_geometry=True,
return_all_records=True).df
geometry_type = df_after.geometry_type
sq = df_before[df_before.geometry.notnull()].geometry.disjoint(geom) == False
df_before = df_before[sq].copy()
before_count = len(df_before)
sq = df_after[df_after.geometry.notnull()].geometry.disjoint(geom) == False
df_after = df_after[sq].copy()
after_count = len(df_after)
geoms_after = df_after.clip(geom.extent)
geoms_before = df_before.clip(geom.extent)
geoms_before_sdf = SpatialDataFrame(geometry=geoms_before)
geoms_after_sdf = SpatialDataFrame(geometry=geoms_after)
q_after = geoms_after_sdf.geometry.JSON == '{"paths":[]}'
geoms_after_sdf = geoms_after_sdf[~q_after].copy()
geoms_after_sdf.reset_index(inplace=True, drop=True)
q_before = geoms_before_sdf.geometry.JSON == '{"paths":[]}'
geoms_before_sdf = geoms_before_sdf[~q_before].copy()
geoms_before_sdf.reset_index(inplace=True, drop=True)
if geometry_type == "Polygon":
before_val = geoms_before_sdf.geometry.get_area('GEODESIC','SQUAREKILOMETERS').sum()
after_val = geoms_after_sdf.geometry.get_area('GEODESIC','SQUAREKILOMETERS').sum()
if after_val > 0:
score = get_score(ratio=before_val/after_val,
baseVal=before_val,
inputVal=after_val)
else:
score = get_score(0, before_val, after_val)
out_sdf[FIELDS[0]][0] = round(before_val,1)
out_sdf[FIELDS[1]][0] = round(after_val,1)
out_sdf[FIELDS[3]][0] = round(before_val - after_val,1)
out_sdf[FIELDS[2]][0] = score
elif geometry_type == "Polyline":
before_val = geoms_before_sdf.geometry.get_length('GEODESIC','KILOMETERS').sum()
after_val = geoms_after_sdf.geometry.get_length('GEODESIC','KILOMETERS').sum()
if after_val > 0:
score = get_score(ratio=before_val/after_val,
baseVal=before_val,
inputVal=after_val)
else:
score = get_score(0, before_val, after_val)
out_sdf[FIELDS[0]][0] = round(before_val,1)
out_sdf[FIELDS[1]][0] = round(after_val,1)
out_sdf[FIELDS[3]][0] = round(before_val - after_val,1)
out_sdf[FIELDS[2]][0] = score
else:
before_count = len(geoms_before_sdf)
after_count = len(geoms_after_sdf)
if after_count > 0:
score = get_score(ratio=before_count/after_count,
baseVal=before_count,
inputVal=after_count)
else:
score = get_score(ratio=0,
baseVal=before_count,
inputVal=after_count)
out_sdf[FIELDS[0]][0] = before_count
out_sdf[FIELDS[1]][0] = after_count
out_sdf[FIELDS[3]][0] = before_count - after_count
out_sdf[FIELDS[2]][0] = score
del sq
del df_after
del df_before
del geom
return out_sdf, out_fl
#arcpy.SetParameterAsText(4, out_grid)
except FunctionError as f_e:
messages = f_e.args[0]
except:
line, filename, synerror = trace()
#--------------------------------------------------------------------------
##if __name__ == "__main__":
## #env.overwriteOutput = True
## argv = tuple(arcpy.GetParameterAsText(i)
## for i in range(arcpy.GetArgumentCount()))
## main(*argv)