def to_geometry(source):
if source is None:
return None
if isinstance(source, Geometry):
return source
if isinstance(source, arcpy.Geometry):
return Geometry(source.JSON)
return Geometry(source)
def validate_geo_gis(geo_gis):
try:
enrich([Geometry({"x": -122.435, "y": 37.785})], gis=geo_gis)
except RuntimeError:
raise Exception(
'GIS Does Not Support GeoEnrichment: {}'.format(geo_gis))
def to_pandas(self, where=None, fields="*", ftype=None):
"""
Exports a table to a Pandas' DataFrame.
=============== ===============================================
**Arguements** **Description**
--------------- -----------------------------------------------
where Optional String. Optional Sql where clause.
--------------- -----------------------------------------------
fields Optional List. The default is all fields (*).
A list of fields can be provided to limit the
data that is returned.
--------------- -----------------------------------------------
ftype Optional String. This value can be dataframe
format type. The value can be None or esri.
+ None - means the dataframe will be a raw view of the table.
+ esri - means the dataframe will be a spatially enable dataframe. (Requires Python API for ArcGIS)
=============== ===============================================
:returns: pd.DataFrame
"""
import pandas as pd
if isinstance(fields, (list, tuple)):
if "OBJECTID" not in fields:
fields.append("OBJECTID")
fields = ",".join(fields)
if where is None:
query = """SELECT {fields} from {tbl} """.format(
tbl=self._table_name, fields=fields)
else:
query = """SELECT {fields} from {tbl} WHERE {where}""".format(
tbl=self._table_name, fields=fields, where=where)
if ftype is None:
return pd.read_sql_query(query, self._con)
elif str(ftype).lower() == 'esri':
try:
from arcgis.geometry import Geometry
from arcgis.features import GeoAccessor, GeoSeriesAccessor
df = pd.read_sql_query(query, self._con)
fields = list(self.fields.keys())
lower_fields = [str(fld).lower() for fld in fields]
if "shape" in lower_fields:
idx = lower_fields.index("shape")
SHAPE = fields[idx]
df[SHAPE] = df[SHAPE].apply(lambda x: Geometry(x[8:]))
df.spatial.set_geometry(SHAPE)
try:
df.spatial.project(self.wkid)
except:
print('nope')
return df
except ImportError:
raise Exception(
"The Python API for ArcGIS is required to import using ftype `esri`"
)
except Exception as e:
raise Exception(e)
def __setitem__(self, key, value):
if value is None or \
(isinstance(value, str) and value == ""):
self.data[key] = value
else:
value = Geometry(value)
self.data[key] = value
def _from_xy(df, x_column, y_column, sr=None):
"""
"""
from arcgis.geometry import SpatialReference, Geometry
from arcgis.features import SpatialDataFrame
if sr is None:
sr = SpatialReference({'wkid': 4326})
if not isinstance(sr, SpatialReference):
if isinstance(sr, dict):
sr = SpatialReference(sr)
elif isinstance(sr, int):
sr = SpatialReference({'wkid': sr})
elif isinstance(sr, str):
sr = SpatialReference({'wkt': sr})
geoms = []
for idx, row in df.iterrows():
geoms.append(
Geometry({
'x': row[x_column],
'y': row[y_column],
'spatialReference': sr
}))
df['SHAPE'] = geoms
return SpatialDataFrame(data=df, sr=sr)
def _get_weighted_centroid_geometry(sub_df: pd.DataFrame, weighting_column: str, sptl_ref: SpatialReference,
geom_col: str = 'SHAPE') -> tuple:
"""
Helper function to calculate centroid coordinates.
Args:
sub_df: DataFrame to calculate weighted coordinates for.
weighting_column: Column to be used for weighting.
sptl_ref: Spatial reference for the output geometry.
geom_col: Optional - Geometry column, defaults to 'SHAPE'.
Returns: Tuple of weighted centroid Geometry objects.
"""
# check if the weighting sum will be naught
wgt_sum = sub_df[weighting_column].sum()
# if there is a weighting sum, use it
if wgt_sum > 0:
wgt_x = np.average(sub_df[geom_col].apply(lambda geom: geom.centroid[0]), weights=sub_df[weighting_column])
wgt_y = np.average(sub_df[geom_col].apply(lambda geom: geom.centroid[1]), weights=sub_df[weighting_column])
# if there is not a weighting sum, just get the average centroid
else:
wgt_x = np.average(sub_df[geom_col].apply(lambda geom: geom.centroid[0]))
wgt_y = np.average(sub_df[geom_col].apply(lambda geom: geom.centroid[1]))
# create a point geometry at the weighted centroid
wgt_geom = Geometry({'x': wgt_x, 'y': wgt_y, 'spatialReference': sptl_ref})
return wgt_geom
def weighted_polygon_centroid(poly_df, wgt_df, poly_id_fld='ID'):
# create a spatial index for both spatially enabled dataframes to speed up the join
if poly_df.spatial._sindex is None:
poly_df.spatial.sindex()
if wgt_df.spatial._sindex is None:
wgt_df.spatial.sindex()
# perform a spatial join between the points and the containing polygons
join_df = wgt_df.spatial.join(poly_df)
# extract the respective coordinates out of the geometry
join_df['x'] = join_df['SHAPE'].apply(lambda geom: geom.centroid[0])
join_df['y'] = join_df['SHAPE'].apply(lambda geom: geom.centroid[1])
# get the id field following the join
join_id_fld = poly_id_fld if poly_id_fld in join_df.columns else f'{poly_id_fld}_right'
# calcuate the weighted centroid coordinates for each polygon and create geometries using these
mean_df = join_df[[join_id_fld, 'x', 'y']].groupby(join_id_fld).mean()
mean_df['SHAPE'] = mean_df.apply(lambda row: Geometry({
'x': row['x'],
'y': row['y'],
'spatialReference': {
'wkid': 4326
}
}),
axis=1)
# clean up the columns to a standard output schema
mean_df.reset_index(inplace=True)
mean_df = mean_df[[join_id_fld, 'SHAPE']].copy()
mean_df.spatial.set_geometry('SHAPE')
mean_df.columns = ['ID', 'SHAPE']
return mean_df
def _prep_sdf_for_nearest(input_dataframe: pd.DataFrame, id_column: str):
"""
Given an input Spatially Enabled Dataframe, prepare it to work
well with the nearest solver.
Args:
input_dataframe: Spatially Enabled Dataframe with really
any geometry.
id_column: Field uniquely identifying each of location to
be used for routing to nearest.
Returns: Spatially Enabled Dataframe of points with correct
columns for routing to nearest.
"""
# check inputs
assert isinstance(input_dataframe, pd.DataFrame), f'The input dataframe must be a Pandas DataFrame, not ' \
f'{type(input_dataframe)}.'
# ensure the geometry is set
geom_col_lst = [
c for c in input_dataframe.columns
if input_dataframe[c].dtype.name.lower() == 'geometry'
]
assert len(geom_col_lst) > 0, 'The DataFrame does not appear to have a geometry column defined. This can be ' \
'accomplished using the "df.spatial.set_geometry" method.'
geom_col = geom_col_lst[0]
# ensure the column is in the dataframe columns
assert id_column in input_dataframe.columns, f'The provided id_column, "{id_column}," does not appear to be in ' \
f'the columns [{", ".join(input_dataframe.columns)}]"'
# par down the input dataframe to just the columns needed
input_dataframe = input_dataframe[[id_column, geom_col]].copy()
# rename the columns to follow the schema needed for routing
input_dataframe.columns = ['ID', 'SHAPE']
# ensure the spatial reference is WGS84 - if not, make it so
if input_dataframe.spatial.sr.wkid != 4326:
input_dataframe = input_dataframe.dm.project(4326)
# if the geometry is not points, we still need points, so get the geometric centroids
if input_dataframe.spatial.geometry_type != ['point']:
input_dataframe['SHAPE'] = input_dataframe[geom_col].apply(
lambda geom: Geometry({
'x': geom.centroid[0],
'y': geom.centroid[1],
'spatialReference': geom.spatial_reference
}))
input_dataframe.spatial.set_geometry('SHAPE')
# add a second column for the ID as Name
input_dataframe['Name'] = input_dataframe['ID']
# ensure the geometry is correctly being recognized
input_dataframe.spatial.set_geometry('SHAPE')
# set the order of the columns and return
return input_dataframe[['ID', 'Name', 'SHAPE']]
def _to_geo_array(values):
from ._array import GeoArray, GeoType
if isinstance(values, GeoArray):
return values.data
if not (isinstance(values, np.ndarray) and
values.dtype == GeoType._record_type):
values = [Geometry(v) for v in values]
return np.asarray(values, dtype=GeoType._record_type)
def arcgis_to_pyprt(feature_set):
"""arcgis_to_pyprt(feature_set) -> List[InitialShape]
This function allows converting an ArcGIS FeatureSet into a list of PyPRT InitialShape instances.
You then typically call the ModelGenerator constructor with the return value if this function as parameter.
Parameters:
feature_set: FeatureSet
Returns:
List[InitialShape]
"""
initial_geometries = []
for feature in feature_set.features:
try:
geo = Geometry(feature.geometry)
if geo.type == 'Polygon':
coord = geo.coordinates()[0]
coord_remove_last = coord[:-1]
coord_inverse = np.flip(coord_remove_last, axis=0)
if coord.shape[1] == 2: # we have to add a dimension
coord_inverse[:, 1] *= -1
coord_add_dim = np.insert(coord_inverse, 1, 0, axis=1)
coord_fin = np.reshape(
coord_add_dim,
(1, coord_add_dim.shape[0] * coord_add_dim.shape[1]))
elif coord.shape[1] == 3: # need to swap the 1 and 2 columns
coord_inverse[:, 1] *= -1
coord_swap_dim = coord_inverse.copy()
temp = np.copy(coord_swap_dim[:, 1])
coord_swap_dim[:, 1] = coord_swap_dim[:, 2]
coord_swap_dim[:, 2] = temp
coord_fin = np.reshape(
coord_swap_dim,
(1, coord_swap_dim.shape[0] * coord_swap_dim.shape[1]))
initial_geometry = pyprt.InitialShape(coord_fin.tolist()[0])
initial_geometries.append(initial_geometry)
except:
print("This feature is not valid: ")
print(feature)
print()
return initial_geometries
def to_wkt(source):
if source is None:
return None
if isinstance(source, Geometry):
return source.WKT
if isinstance(source, arcpy.Geometry):
return source.WKT
geom = Geometry(source)
return geom.WKT
def _as_arcgis(self, spatial_reference):
"""Return an arcgis Geometry.
Arguments:
spatial_reference A spatial reference integer code or definition
dictionary, for example {'wkid': 3857}
"""
if isinstance(spatial_reference, int):
spatial_reference = {'wkid': spatial_reference}
if isinstance(self, ShapelyPoint) or \
isinstance(self, ShapelyMultiPoint) or \
isinstance(self, ShapelyLineString) or \
isinstance(self, ShapelyMultiLineString):
geom = Geometry(self.__geo_interface__)
geom['spatialReference'] = spatial_reference
elif isinstance(self, ShapelyPolygon):
linear_rings = [self.exterior]
linear_rings += self.interiors
rings = [
list(r.__geo_interface__['coordinates']) for r in linear_rings
]
geom = Geometry({
'rings': rings,
'spatialReference': spatial_reference
})
elif isinstance(self, ShapelyMultiPolygon):
linear_rings = []
for poly in self.geoms:
linear_rings += [poly.exterior]
linear_rings += poly.interiors
rings = [
list(r.__geo_interface__['coordinates']) for r in linear_rings
]
geom = Geometry({
'rings': rings,
'spatialReference': spatial_reference
})
return geom
def new_dest():
geom = Geometry({
'x': -122.7342423,
'y': 45.4383307,
'spatialReference': {
'latestWkid': 4326,
'wkid': 4326
}
})
return geom
def validate_img_gis(geo_gis, img_url):
try:
img_lyr = ImageryLayer(img_url, gis=geo_gis)
img_lyr.get_samples(Geometry({
"x": -122.435,
"y": 37.785
}),
geometry_type='esriGeometryPoint')
print('Thematic Service GIS: {}'.format(geo_gis))
except RuntimeError:
raise Exception('{} Does Not Support getSamples on Service: {}'.format(
geo_gis, img_url))
def from_featureset(fset, sr=None):
"""
Converts a FeatureSet to a pd.DataFrame
=============== ==============================================
Arguments Description
--------------- ----------------------------------------------
fset Required FeatureSet. FeatureSet object.
=============== ==============================================
return Panda's DataFrame
"""
if isinstance(fset, FeatureSet):
rows = []
sr = fset.spatial_reference
dt_fields = [
fld['name'] for fld in fset.fields
if fld['type'] == 'esriFieldTypeDate'
]
if sr is None:
sr = {'wkid': 4326}
for feat in fset.features:
a = feat.attributes
if not feat.geometry is None:
g = feat.geometry
g['spatialReference'] = sr
a['SHAPE'] = Geometry(g)
rows.append(a)
del a, feat
from arcgis.features import GeoAccessor, GeoSeriesAccessor
df = pd.DataFrame(data=rows)
for fld in dt_fields:
try:
df[fld] = pd.to_datetime(df[fld] / 1000,
infer_datetime_format=True,
unit='s')
except:
df[fld] = pd.to_datetime(df[fld], infer_datetime_format=True)
if 'SHAPE' in df.columns:
df.spatial.set_geometry("SHAPE")
df.spatial.sr = sr
return df
else:
return None
def get_esri_geometry_for_h3_id(h3_id: str) -> Polygon:
"""
Convert an Uber H3 id to an ArcGIS Python API Polygon Geometry object.
:param h3_id:String Uber H3 id.
:return: ArcGIS Python API Polygon Geometry object
"""
# get a list of coordinate rings for the hex id's
coord_lst = [h3.h3_to_geo_boundary(h3_id, geo_json=True)]
# creat a geometry object using this geometry list
return Geometry({
"type": "Polygon",
"coordinates": coord_lst,
'spatialReference': {
'wkid': 4326
}
})
def soil_type(self):
parcel_geometry = wkt.loads(self.transformed.wkt)
geometry_map = mapping(parcel_geometry)
esriPolygon = {
"spatialReference": {
"wkid": 31370
},
"rings":
[[list(p) for p in r] for r in geometry_map["coordinates"]]
}
response = requests.get(
f"https://geoservices.wallonie.be/arcgis/rest/services/SOL_SOUS_SOL/CNSW__PRINC_TYPES_SOLS/MapServer/identify?f=json&geometry={esriPolygon}&tolerance=0&mapExtent={parcel_geometry.bounds}&sr=31370&imageDisplay=1,1,1&layers=all&geometryType=esriGeometryPolygon&geometryPrecision=4"
)
if not response.ok:
raise 'Could not define the soil type'
if "error" in response.json().keys():
raise response.json()
soil_type_data = {}
for data in response.json()["results"]:
geometry = wkt.loads(Geometry(data["geometry"]).WKT)
code = data["attributes"]["CODE"]
soil_type_data[code] = {
"code": code,
"description": data["attributes"]["DESCRIPTION"],
"area": 0
}
geometry = cascaded_union(geometry).buffer(0)
intersection = geometry.intersection(parcel_geometry)
soil_type_data[code]["area"] += intersection.area
for val in soil_type_data.values():
val["proportion"] = val["area"] / parcel_geometry.area
return soil_type_data.values()
def prep_sdf_for_nearest(df, id_fld, weighting_points=None):
"""
Given an input Spatially Enabled Dataframe, prepare it to work well with the nearest solver.
:param df: Spatially Enabled Dataframe with really any geometry.
:param id_fld: Field uniquely identifying each of location to be used for routing to nearest.
:param weighting_points: Spatially Enabled Dataframe of points for calculating weighted centroids.
:return: Spatially Enabled Dataframe of points with correct columns for routing to nearest.
"""
# par down the input dataframe to just the columns needed
df = df[[id_fld, 'SHAPE']].copy()
# rename the columns to follow the schema needed for routing
df.columns = ['ID', 'SHAPE']
# if the geometry is polygons and there is a weighting points spatially enabled dataframe
if df.spatial.geometry_type == ['polygon'
] and weighting_points is not None:
# calculate the weighted centroid for the polygons
df = weighted_polygon_centroid(df, weighting_points)
# otherwise, if the geometry is not points, we still need points, so just get the geometric centroids
# TODO: Account for polygons NOT always being in WGS 84
elif df.spatial.geometry_type != ['point'] and weighting_points is None:
df['SHAPE'] = df['SHAPE'].apply(
lambda geom: Geometry({
'x': geom.centroid[0],
'y': geom.centroid[1],
'spatialReference': {
'wkid': 4326
}
}))
# add a second column for the ID as Name
df['Name'] = df['ID']
# ensure the geometry is correctly being recognized
df.spatial.set_geometry('SHAPE')
# set the order of the columns and return
return df[['ID', 'Name', 'SHAPE']].copy()
def set_features(self):
df_list = []
for idx, row in enumerate(
self.grid_sdf.iterrows()): #self.grid_sdf.iterrows()):
geom = Geometry(row[1].SHAPE)
print(idx)
sp_filter = filters.intersects(geom, self.grid_wkid)
data_fl = FeatureLayer(url=self.feat_url, gis=self.gis_conn)
## Change return all records to True
df_list.append(
data_fl.query(geometry_filter=sp_filter,
return_all_records=self.return_all_records).df)
self.features = df_list
def _convert_geometry_column_to_geometry(df, geometry_column):
"""
Helper function to follow the paradigm of DRYD (don't repeat yourself, DUMMY!) for
converting a column from object (str) to a valid Geometry type, and enabling
this column as the recognized geometry column for a fully functional Spatially
Enabled Dataframe.
Args:
df: Pandas DataFrame with a column containing valid Esri JSON objects
describing Geometries.
geometry_column: Column containing the valid Esri Geometry object instances.
Returns: Spatially Enabled Pandas DataFrame
"""
# convert the values in the geometry column to geometry objects
df[geometry_column] = df[geometry_column].apply(
lambda val: Geometry(json.loads(val)))
# tell the GeoAccessor to recognize the column
df.spatial.set_geometry(geometry_column)
return df
def split_at_point(self, point_geometry):
"""
Returns two polyline geometry objects as a list split at the intersection of the line.
:param point_geometry: Required arcgis.geometry.Point
ArcGIS Point geometry defining the location the line will be split at.
:return: Two item list of arcgis.geometry.Polyline objects
List of two ArcGIS Polyline objects, one on either side of the input point location.
"""
if not isinstance(self, Polyline):
raise Exception('Split at point can only be performed on a Polyline geometry object.')
if not isinstance(point_geometry, Point):
raise Exception('Split at point requires a Point geometry object to define the split location.')
if self.spatial_reference is None:
raise Warning('The spatial reference for the line to be split is not defined.')
if point_geometry.spatial_reference is None:
raise Warning('The spatial reference of the point defining the split location is not defined.')
if (self.spatial_reference != point_geometry.spatial_reference and
self.spatial_reference.wkid != point_geometry.spatial_reference.wkid and
self.spatial_reference.latestWkid != point_geometry.spatial_reference.wkid and
self.spatial_reference.wkid != point_geometry.spatial_reference.latestWkid and
self.spatial_reference.latestWkid != point_geometry.spatial_reference.latestWkid):
raise Exception('The spatial reference for the line and point are not the same.')
# if HASARCPY:
# raise Exception('Not yet implemented')
if HASSHAPELY:
linestring_geometry = self.as_shapely
point_geometry = point_geometry.as_shapely
split_result = ops.split(linestring_geometry, point_geometry)
polyline_list = [Geometry({
'paths': [line_string.__geo_interface__['coordinates']],
'spatialReference': self.spatial_reference})
for line_string in split_result]
return polyline_list
else:
raise Exception('Shapely is required to perform split_at_point')
def _from_xy(df, x_column, y_column, sr=None):
"""
Takes an X/Y Column and Creates a Point Geometry from it.
"""
from arcgis.geometry import SpatialReference, Geometry
if sr is None:
sr = SpatialReference({'wkid' : 4326})
if not isinstance(sr, SpatialReference):
if isinstance(sr, dict):
sr = SpatialReference(sr)
elif isinstance(sr, int):
sr = SpatialReference({'wkid' : sr})
elif isinstance(sr, str):
sr = SpatialReference({'wkt' : sr})
geoms = []
for idx, row in df.iterrows():
geoms.append(
Geometry({'x' : row[x_column], 'y' : row[y_column],
'spatialReference' : sr})
)
df['SHAPE'] = geoms
df.spatial.set_geometry('SHAPE')
return df
def set_selected(self, indicator):
created = False
out_sdf = None
# Indicator Feature Layer
indicator_url = self.__getattribute__(indicator + '_url')
print(indicator_url)
#data_fl = FeatureLayer(url=indicator_url, gis=self.gis_conn)
# Enumerate Used to Leverage the Merge Method on the Data Frame.
# Set the First and Merge the Remainder to the First.
for idx, row in enumerate(self.grid_sdf.iterrows()):
# Negative Buffer Used to Avoid Selecting More Than 1 Cell
sp_filter = filters.intersects(
Geometry(row[1].SHAPE).buffer(-.1), self.grid_wkid)
if not indicator_url:
df_current = SpatialDataFrame(
columns=field_schema.get(indicator),
geometry=[Geometry(json.loads(row[1].SHAPE.JSON))])
created = True
else:
# Negative Buffer to Select a Single Grid Cell
sp_filter = filters.intersects(
Geometry(row[1].SHAPE).buffer(-.1), self.grid_wkid)
df_current = data_fl.query(
geometry_filter=sp_filter,
return_all_records=self.return_all_records).df
# Set The First Instance
if idx == 0:
# Check If Cell Found in Target Indicator Layer
if df_current.empty:
# Use Grid SDF Row Geom to Insert Empty Record for Target Indicator
data_fl.edit_features(
adds=[{
'attributes': {},
'geometry': json.loads(row[1].SHAPE.JSON)
}])
# Select Newly Created Cell As Input
out_sdf = data_fl.query(
geometry_filter=sp_filter,
return_all_records=self.return_all_records).df
else:
# Use Matched Grid Cell
out_sdf = df_current
# Append Additional Data
else:
# Check If Cell Found in Target Indicator Layer
if df_current.empty:
# Use Grid SDF Row Geom to Insert Empty Record for Target Indicator
data_fl.edit_features(
adds=[{
'attributes': {},
'geometry': json.loads(row[1].SHAPE.JSON)
}])
# Select Newly Created Cell As Input
out_sdf.merge(
data_fl.query(
geometry_filter=sp_filter,
return_all_records=self.return_all_records).df)
else:
# Use Matched Grid Cell
out_sdf = out_sdf.merge_datasets(df_current)
self.selected = out_sdf.reset_index(drop=False)
print("Selected: " + str(len(out_sdf)))
return created
class AccessPutin(unittest.TestCase):
canyon_reach_id = 3066
putin_point = Geometry({
'x': -121.633094,
'y': 45.79532367,
'spatialReference': {
'wkid': 4326
}
})
point_type = 'access'
subtype = 'putin'
name = 'Hayes Creek'
side_of_river = 'left'
collection_method = 'digitized'
collection_date = '02 Nov 1998'
test_series = pd.Series({
"_geometry":
Geometry({
'x': -121.633094,
'y': 45.79532367,
'spatialReference': {
'wkid': 4326
}
}),
"reach_id":
str(canyon_reach_id),
"point_type":
point_type,
"subtype":
subtype,
"name":
name,
"side_of_river":
side_of_river,
"nhdplus_measure":
None,
"nhdplus_reach_id":
None,
"collection_method":
collection_method,
"collection_date":
collection_date,
"notes":
None,
"description":
None,
"type":
point_type
})
test_feature_dict = {
"geometry": {
"x": -121.633094,
'y': 45.79532367,
"spatialReference": {
"wkid": 4326
}
},
"attributes": {
'reach_id': '3066',
'point_type': 'access',
'subtype': 'putin',
'name': 'Hayes Creek',
'side_of_river': 'left',
'nhdplus_measure': None,
'nhdplus_reach_id': None,
'collection_method': 'digitized',
'update_date': '02 Nov 1998',
'notes': None,
'description': None,
'difficulty': None
}
}
test_snap_geom_dict = {
'x': -121.63309439504,
'y': 45.7953235252763,
'spatialReference': {
'wkid': 4326
}
}
def test_instantiate_access(self):
access = ReachPoint(reach_id=self.canyon_reach_id,
geometry=self.putin_point,
point_type=self.point_type,
subtype=self.subtype,
name=self.name,
side_of_river=self.side_of_river,
collection_method=self.collection_method,
update_date=self.collection_date)
self.assertEqual(type(access), ReachPoint)
def test_as_feature(self):
access = ReachPoint(reach_id=self.canyon_reach_id,
geometry=self.putin_point,
point_type=self.point_type,
subtype=self.subtype,
name=self.name,
side_of_river=self.side_of_river,
collection_method=self.collection_method,
update_date=self.collection_date)
delattr(
access,
'uid') # since this will be different every time, just remove it
self.assertDictEqual(access.as_feature.as_dict, self.test_feature_dict)
def test_snap_geom_to_nhdplus(self):
access = ReachPoint(reach_id=self.canyon_reach_id,
geometry=self.putin_point,
point_type=self.point_type,
subtype=self.subtype,
name=self.name,
side_of_river=self.side_of_river,
collection_method=self.collection_method,
update_date=self.collection_date)
access.snap_to_nhdplus()
self.assertDictEqual(access.as_feature.as_dict['geometry'],
self.test_snap_geom_dict)
def process_by_metadata(gis):
return_all_records = False
look_back_days = config.look_back_days
dates = csl.get_dates_in_range(look_back_days)
where_clause = csl.form_query_string(dates)
grid_fl = FeatureLayer(url=config.grid_url)
grid_sdf = grid_fl.query(return_all_records=return_all_records,
where=where_clause).df
geometry = grid_sdf.geometry
sr = {'wkid': 4326}
sp_rel = "esriSpatialRelIntersects"
for idx, row in enumerate(grid_sdf.iterrows()):
geom = row[1].SHAPE
new_geom = Geometry({
"rings":
[[[geom.extent.upperRight.X - .1, geom.extent.lowerLeft.Y + .1],
[geom.extent.lowerLeft.X + .1, geom.extent.lowerLeft.Y + .1],
[geom.extent.lowerLeft.X + .1, geom.extent.upperRight.Y - .1],
[geom.extent.upperRight.X - .1, geom.extent.upperRight.Y - .1],
[geom.extent.upperRight.X - .1, geom.extent.lowerLeft.Y + .1]]],
"spatialReference": {
"wkid": 4326
}
})
grid_filter = filters._filter(new_geom, sr, sp_rel)
sp_filter = filters._filter(geom, sr, sp_rel)
data_fl = FeatureLayer(url=config.features_url)
#out_fields=in_fields,
data_sdf = data_fl.query(geometry_filter=sp_filter,
return_geometry=True,
return_all_records=return_all_records).df
print('Processing Completeness')
#bounding_box = '(37.708132, -122.513617, 37.832132, -122.349607)'
bounding_box = '(' + \
str(geom.extent.lowerLeft.Y) + ',' + \
str(geom.extent.lowerLeft.X) + ',' + \
str(geom.extent.upperRight.Y) + ',' + \
str(geom.extent.upperRight.X) + ')'
osm_sdf = runner.gen_osm_sdf('line',
bounding_box,
osm_tag='highway',
present=True)
completeness_sdf, completeness_fl = comp.completeness(
gis, osm_sdf, data_sdf, config.completeness_url, grid_filter, geom)
print(completeness_sdf)
#update_features(them_acc_sdf, them_acc_fl)
print('Completeness Updated')
print('Processing Logical Consistency')
lc_sdf, lc_fl = lc.logical_consisitency(
gis, config.template_fc, config.template_gdb,
config.attr_check_file, config.attr_check_tab, data_sdf,
config.features_url, config.logical_consistency_url, grid_filter,
geom, config.attr_error_field_count, config.attr_error_field_def)
print(lc_sdf)
update_features(lc_sdf, lc_fl)
print('Logical Consistency Updated.')
print('Processing temporal currency')
tc_sdf, tc_fl = tc.temporal_currency(gis, data_sdf,
config.currency_url, grid_filter,
geom, config.currency_field)
print(tc_sdf)
#update_features(tc_sdf, tc_fl)
print('Temporal Currency Updated')
print('Processing source lineage')
sl_sdf, sl_fl = sl.source_lineage(gis, data_sdf,
config.source_lineage_url,
grid_filter, geom,
config.search_field,
config.value_field)
print(sl_sdf)
#update_features(sl_sdf, sl_fl)
print('Source Lineage Updated')
print('Processing Positional Accuracy')
pa_sdf, pa_fl = pa.positional_accuracy(gis, data_sdf,
config.positional_acc_url,
grid_filter, geom,
config.positional_acc_field)
print(pa_sdf)
#update_features(pa_sdf, pa_fl)
print('Positional Accuracy Updated')
print('Processing Thematic Accuracy')
them_acc_sdf, them_acc_fl = them_acc.thematic_accuracy(
gis, data_sdf, config.thematic_url, grid_filter, geom,
config.thematic_acc_field)
print(them_acc_sdf)
#update_features(them_acc_sdf, them_acc_fl)
print('Theamatic Accuracy Updated')
return
col='ST') #column to get unique values from
# In[11]:
item = gis.content.get("8444e275037549c1acab02d2626daaee")
flayer = item.layers[0]
df2 = flayer.query().sdf
# In[12]:
fset = flayer.query()
# In[13]:
from arcgis.geometry import Geometry
g = Geometry(fset.features[0].geometry)
g.as_arcpy
# In[14]:
#create an opacity visual variable based on a percent of dominant parties in registered citizens.
opacity_expression = (
"var republican = $feature.MP06025a_B;var democrat = $feature.MP06024a_B;"
"var independent = $feature.MP06026a_B;var parties = [republican, democrat, independent];"
"var total = Sum(parties);var max = Max(parties);return (max / total) * 100;"
)
opacity_stops = [{
"value": 33,
"transparency": 0.05 * 255,
"label": "< 33%"
}, {
def get_dataframe(in_features, gis=None):
"""
Get a spatially enabled dataframe from the input features provided.
:param in_features: Spatially Enabled Dataframe | String path to Feature Class | pathlib.Path object to feature
class | ArcGIS Layer object |String url to Feature Service | String Web GIS Item ID
Resource to be evaluated and converted to a Spatially Enabled Dataframe.
:param gis: Optional GIS object instance for connecting to resources.
"""
# if a path object, convert to a string for following steps to work correctly
in_features = str(in_features) if isinstance(in_features, pathlib.Path) else in_features
# helper for determining if feature layer
def _is_feature_layer(in_ftrs):
if hasattr(in_ftrs, 'isFeatureLayer'):
return in_ftrs.isFeatureLayer
else:
return False
# if already a Spatially Enabled Dataframe, mostly just pass it straight through
if isinstance(in_features, pd.DataFrame) and in_features.spatial.validate() is True:
df = in_features
# if a csv previously exported from a Spatially Enabled Dataframe, get it in
elif isinstance(in_features, str) and os.path.exists(in_features) and in_features.endswith('.csv'):
df = pd.read_csv(in_features)
df['SHAPE'] = df['SHAPE'].apply(lambda geom: Geometry(eval(geom)))
# this almost always is the index written to the csv, so taking care of this
if df.columns[0] == 'Unnamed: 0':
df = df.set_index('Unnamed: 0')
del (df.index.name)
# create a Spatially Enabled Dataframe from the direct url to the Feature Service
elif isinstance(in_features, str) and in_features.startswith('http'):
# submitted urls can be lacking a few essential pieces, so handle some contingencies with some regex matching
regex = re.compile(r'((^https?://.*?)(/\d{1,3})?)\?')
srch = regex.search(in_features)
# if the layer index is included, still clean by dropping any possible trailing url parameters
if srch.group(3):
in_features = f'{srch.group(1)}'
# ensure at least the first layer is being referenced if the index was forgotten
else:
in_features = f'{srch.group(2)}/0'
# if the layer is unsecured, a gis is not needed, but we have to handle differently
if gis is not None:
df = FeatureLayer(in_features, gis).query(out_sr=4326, as_df=True)
else:
df = FeatureLayer(in_features).query(out_sr=4326, as_df=True)
# create a Spatially Enabled Dataframe from a Web GIS Item ID
elif isinstance(in_features, str) and len(in_features) == 32:
# if publicly shared on ArcGIS Online this anonymous gis can be used to access the resource
if gis is None:
gis = GIS()
itm = gis.content.get(in_features)
df = itm.layers[0].query(out_sr=4326, as_df=True)
elif isinstance(in_features, (str, pathlib.Path)):
df = GeoAccessor.from_featureclass(in_features)
# create a Spatially Enabled Dataframe from a Layer
elif _is_feature_layer(in_features):
df = FeatureSet.from_json(arcpy.FeatureSet(in_features).JSON).sdf
# sometimes there is an issue with modified or sliced dataframes with the SHAPE column not being correctly
# recognized as a geometry column, so try to set it as the geometry...just in case
elif isinstance(in_features, pd.DataFrame) and 'SHAPE' in in_features.columns:
in_features.spatial.set_geometry('SHAPE')
df = in_features
if df.spatial.validate() is False:
raise Exception('Could not process input features for get_dataframe function. Although the input_features '
'appear to be in a Pandas Dataframe, the SHAPE column appears to not contain valid '
'geometries. The Dataframe is not validating using the *.spatial.validate function.')
else:
raise Exception('Could not process input features for get_dataframe function.')
# ensure the universal spatial column is correctly being recognized
df.spatial.set_geometry('SHAPE')
return df
def thematic_accuracy(out_sdf, df_list, f_thm_acc, them_gis, them_url):
print('Running Thematic Accuracy')
f_thm_acc = validate_field(f_thm_acc, list(df_list[0]))
# List Used for Logging Differences in Population Sources
pop_diff = []
for idx, row in enumerate(out_sdf.iterrows()):
df_current = df_list[idx]
##-----------------------------------------------------------------------------
## Uses Geoenrichment - Not available outside of AGOL
# Pull GeoEnrichment Figures
# enriched = enrich([row[1]['SHAPE']], gis=geo_gis)
# if 'TOTPOP' not in list(enriched):
# enriched_pop = -1
# else:
# enriched_pop = enriched.TOTPOP[0]
#
# # Pull Samples From Configured Population Service
# img_lyr = ImageryLayer(img_url, gis=geo_gis)
# cells = img_lyr.properties.maxImageHeight * img_lyr.properties.maxImageWidth
# samples = img_lyr.get_samples(
# row[1]['SHAPE'],
# geometry_type='esriGeometryPolygon',
# sample_count=cells
# )
# sample_total = sum([int(sample['value']) for sample in samples])
#
# # Push Significant Values Into List for Averaging
# if enriched_pop or sample_total < 100:
# pass
# else:
# diff = abs(enriched_pop - sample_total)
# if diff > 100:
# pop_diff.append(diff)
#
# tot_pop = enriched_pop if enriched_pop > 0 else sample_total
# tot_pop = tot_pop if tot_pop > 0 else -1
##-----------------------------------------------------------------------------
them_lyr = FeatureLayer(url=them_url, gis=them_gis)
geom = Geometry(row[1].SHAPE).buffer(-.01)
sp_filter = filters.intersects(geom, 4326)
them_sdf = them_lyr.query(geometry_filter=sp_filter,
return_all_records=True).df
#print(them_sdf)
if len(df_current) > 0:
count = len(df_current)
max_val = df_current[f_thm_acc].max()
max_scale = 100 * (
len(df_current[df_current[f_thm_acc] == max_val]) / count)
min_val = df_current[f_thm_acc].min()
min_scale = 100 * (
len(df_current[df_current[f_thm_acc] == min_val]) / count)
vc = df_current[f_thm_acc].value_counts()
common = df_current[f_thm_acc].mode() # Used in MSP
mean = df_current[f_thm_acc].mean()
if len(common) > 0:
common = common[0]
common_count = vc[common]
common_per = (vc[common] / count) * 100
else:
common = min_val
common_count = 1
common_per = 100
count_2500 = 0
count_5000 = 0
count_12500 = 0
count_25000 = 0
count_50000 = 0
count_100000 = 0
count_250000 = 0
count_500000 = 0
count_1000000 = 0
if 2500 in vc:
count_2500 = vc[2500]
if 5000 in vc:
count_5000 = vc[5000]
if 12500 in vc:
count_12500 = vc[12500]
if 25000 in vc:
count_25000 = vc[25000]
if 50000 in vc:
count_50000 = vc[50000]
if 100000 in vc:
count_100000 = vc[100000]
if 250000 in vc:
count_250000 = vc[250000]
if 500000 in vc:
count_500000 = vc[500000]
if 1000000 in vc:
count_1000000 = vc[1000000]
MSP = get_msp(scale=common) # SHOULD UPDATE MISSION_PLANNING FIELD
if not out_sdf['MEAN'][0]:
m = 0
else:
m = out_sdf['MEAN'][0]
SCORE_VALUE = them_sdf['grls_score'].loc[
0] #get_equal_breaks_score(m)# get_equal_breaks_score(output_features, ['MEAN','EQUAL']) # PUT SCORE IN EQUAL
#GRLS = SCORE_VALUE
#domScale = common
# FIELD 1 is the source, Field 2 is the field to be updated
#df_current['EQUAL'] = SCORE_VALUE # ASSIGNS EQUAL TO LANSCAN_SCALE
#29 field
out_sdf.set_value(idx,
field_schema.get('them')[0], common) # median
out_sdf.set_value(idx,
field_schema.get('them')[1],
common_count) # % common
out_sdf.set_value(idx,
field_schema.get('them')[2],
round(common_per, 1))
out_sdf.set_value(idx, field_schema.get('them')[3], min_val)
out_sdf.set_value(idx,
field_schema.get('them')[4], round(min_scale, 1))
out_sdf.set_value(idx, field_schema.get('them')[5], max_val)
out_sdf.set_value(idx,
field_schema.get('them')[6], round(max_scale, 1))
out_sdf.set_value(idx, field_schema.get('them')[7], count_2500)
out_sdf.set_value(idx, field_schema.get('them')[8], count_5000)
out_sdf.set_value(idx, field_schema.get('them')[9], count_12500)
out_sdf.set_value(idx, field_schema.get('them')[10], count_25000)
out_sdf.set_value(idx, field_schema.get('them')[11], count_50000)
out_sdf.set_value(idx, field_schema.get('them')[12], count_100000)
out_sdf.set_value(idx, field_schema.get('them')[13], count_250000)
out_sdf.set_value(idx, field_schema.get('them')[14], count_500000)
out_sdf.set_value(idx, field_schema.get('them')[15], count_1000000)
out_sdf.set_value(idx,
field_schema.get('them')[16],
round(count_2500 * 100 / count, 1))
out_sdf.set_value(idx,
field_schema.get('them')[17],
round(count_5000 * 100 / count, 1))
out_sdf.set_value(idx,
field_schema.get('them')[18],
round(count_12500 * 100 / count, 1))
out_sdf.set_value(idx,
field_schema.get('them')[19],
round(count_25000 * 100 / count, 1))
out_sdf.set_value(idx,
field_schema.get('them')[20],
round(count_50000 * 100 / count, 1))
out_sdf.set_value(idx,
field_schema.get('them')[21],
round(count_100000 * 100 / count, 1))
out_sdf.set_value(idx,
field_schema.get('them')[22],
round(count_250000 * 100 / count, 1))
out_sdf.set_value(idx,
field_schema.get('them')[23],
round(count_500000 * 100 / count, 1))
out_sdf.set_value(idx,
field_schema.get('them')[24],
round(count_1000000 * 100 / count, 1))
out_sdf.set_value(idx, field_schema.get('them')[25], count)
out_sdf.set_value(idx,
field_schema.get('them')[26],
str(MSP)) #MISSION_PLANNING FIELD
out_sdf.set_value(idx,
field_schema.get('them')[27],
SCORE_VALUE) #), # THEMATIC SCALE VALUE
#out_sdf.set_value(idx, field_schema.get('them')[27], tot_pop) # ), # THEMATIC SCALE VALUE
out_sdf.set_value(idx,
field_schema.get('them')[28],
population_scale(
common, SCORE_VALUE)) # POPULATION_SCALE
out_sdf.set_value(idx, field_schema.get('them')[29], mean)
#to 28
else:
for i in range(0, 25):
out_sdf.set_value(idx, field_schema.get('them')[i], -1)
out_sdf.set_value(idx, field_schema.get('them')[25], 0)
out_sdf.set_value(idx, field_schema.get('them')[26], 'N/A')
out_sdf.set_value(idx, field_schema.get('them')[27], 'N/A')
out_sdf.set_value(idx, field_schema.get('them')[28], 0)
out_sdf.set_value(idx, field_schema.get('them')[29], -1)
del df_current
#print('Average Difference of Population Estimates: {}'.format(np.average(pop_diff)))
return out_sdf
def completeness(out_sdf, df_list, osm_sdf):
print('Running Completeness')
for idx, row in enumerate(out_sdf.iterrows()):
before_val = None
geom = Geometry(row[1].SHAPE)
# Unpack Geom Extent as OSM Expects
bbox = (geom.extent[1], geom.extent[0], geom.extent[3], geom.extent[2])
# Fetch OSM SpatialDataFrame
osm_sdf = gen_osm_sdf('line', bbox, osm_tag='highway')
data_sdf = df_list[idx]
if len(data_sdf) == 0:
before_val = 0
else:
sq = data_sdf[data_sdf.geometry.notnull()].geometry.disjoint(
geom) == False
df_before = data_sdf[sq].copy()
geoms_before = df_before.clip(geom.extent)
geoms_before_sdf = SpatialDataFrame(geometry=geoms_before)
q_before = geoms_before_sdf['SHAPE'] == {"paths": []}
geoms_before_sdf = geoms_before_sdf[~q_before].copy()
geoms_before_sdf.reset_index(inplace=True, drop=True)
geometry_type = osm_sdf.geometry_type
sq = osm_sdf[osm_sdf.geometry.notnull()].geometry.disjoint(
geom) == False
df_after = osm_sdf[sq].copy()
geoms_after = df_after.clip(geom.extent)
geoms_after_sdf = SpatialDataFrame(geometry=geoms_after)
#geoms_after_sdf = SpatialDataFrame({'Pass': '******'}, geometry=geoms_after, index=[0])
q_after = geoms_after_sdf['SHAPE'] == {"paths": []}
geoms_after_sdf = geoms_after_sdf[~q_after].copy()
geoms_after_sdf.reset_index(inplace=True, drop=True)
# This Need Work
if geometry_type == "Polygon":
if before_val == None:
before_val = geoms_before_sdf.geometry.project_as(
4326).get_area('GEODESIC', 'SQUAREKILOMETERS').sum()
after_val = geoms_after_sdf.geometry.project_as(4326).get_area(
'GEODESIC', 'SQUAREKILOMETERS').sum()
if after_val > 0:
score = get_cp_score(ratio=before_val / after_val,
baseVal=before_val,
inputVal=after_val)
else:
score = get_cp_score(0, before_val, after_val)
out_sdf.set_value(idx,
field_schema.get('cmpl')[0],
round(before_val, 1))
out_sdf.set_value(idx,
field_schema.get('cmpl')[1], round(after_val, 1))
out_sdf.set_value(idx,
field_schema.get('cmpl')[3],
round(before_val - after_val, 1))
out_sdf.set_value(idx, field_schema.get('cmpl')[2], score)
elif geometry_type == "Polyline":
if before_val == None:
geom = geoms_before_sdf.geometry
geom_projected = geoms_before_sdf.geometry.project_as(3857)
before_val = int(sum(geom_projected.length.tolist()))
geom_projected = geoms_before_sdf.geometry.project_as(3857)
after_val = int(sum(geom_projected.length.tolist()))
if after_val > 0:
score = get_cp_score(ratio=before_val / after_val,
baseVal=before_val,
inputVal=after_val)
else:
score = get_cp_score(0, before_val, after_val)
out_sdf.set_value(idx,
field_schema.get('cmpl')[0],
round(before_val, 1))
out_sdf.set_value(idx,
field_schema.get('cmpl')[1], round(after_val, 1))
out_sdf.set_value(idx,
field_schema.get('cmpl')[3],
round(before_val - after_val, 1))
out_sdf.set_value(idx, field_schema.get('cmpl')[2], score)
else:
if before_val == None:
before_count = len(geoms_before_sdf)
else:
before_count = 0
after_count = len(geoms_after_sdf)
if after_count > 0:
score = get_cp_score(ratio=before_count / after_count,
baseVal=before_count,
inputVal=after_count)
else:
score = get_cp_score(ratio=0,
baseVal=before_count,
inputVal=after_count)
out_sdf.set_value(idx, field_schema.get('cmpl')[0], before_count)
out_sdf.set_value(idx, field_schema.get('cmpl')[1], after_count)
out_sdf.set_value(idx,
field_schema.get('cmpl')[3],
before_count - after_count)
out_sdf.set_value(idx, field_schema.get('cmpl')[2], score)
del sq
del df_after
del geom
if before_val != None:
print(before_val)
# del df_before
return out_sdf
x, y = p
if x >= m1 and x <= m2 and y >= n1 and y <= n2:
return True
return False
return (point_in((x1, y1), b)) or (point_in((x2, y1), b)) or (point_in((x2, y2), b)) or (point_in((x1, y2), b))
# iterate through image tiles on naip_image_layer starting from bottom row
for y_idx in range(y_num_tile):
for x_idx in range(x_num_tile):
image_name = str(y_idx*x_num_tile + x_idx)
x_start = min_x + x_idx * tile_size
y_start = min_y + y_idx * tile_size
# export annotations for buildings in the image
tile_image_geometry = Geometry({
"rings" : [[[x_start,y_start],[x_start+tile_size,y_start],[x_start+tile_size,y_start+tile_size],[x_start,y_start+tile_size]]],
"spatialReference" : {"wkid" : crs_id}
})
annotations = []
for idx, bbox in enumerate(building_data.geometry):
# bbox is polygon
bbox_extent = bbox.extent
tile_extent = tile_image_geometry.extent
try:
# if bbox overlaps with tile image extent, record the building bbox
if overlap(bbox_extent, tile_extent):
# bbox contains normalized [xywh]
x1_r, y1_r, x2_r, y2_r = bbox_extent
# clipping
x1 = max(x_start, x1_r)
y1 = max(y_start, y1_r)
x2 = min(x2_r, x_start+tile_size)
