def to_geometry(source): if source is None: return None if isinstance(source, Geometry): return source if isinstance(source, arcpy.Geometry): return Geometry(source.JSON) return Geometry(source)
def validate_geo_gis(geo_gis): try: enrich([Geometry({"x": -122.435, "y": 37.785})], gis=geo_gis) except RuntimeError: raise Exception( 'GIS Does Not Support GeoEnrichment: {}'.format(geo_gis))
def to_pandas(self, where=None, fields="*", ftype=None): """ Exports a table to a Pandas' DataFrame. =============== =============================================== **Arguements** **Description** --------------- ----------------------------------------------- where Optional String. Optional Sql where clause. --------------- ----------------------------------------------- fields Optional List. The default is all fields (*). A list of fields can be provided to limit the data that is returned. --------------- ----------------------------------------------- ftype Optional String. This value can be dataframe format type. The value can be None or esri. + None - means the dataframe will be a raw view of the table. + esri - means the dataframe will be a spatially enable dataframe. (Requires Python API for ArcGIS) =============== =============================================== :returns: pd.DataFrame """ import pandas as pd if isinstance(fields, (list, tuple)): if "OBJECTID" not in fields: fields.append("OBJECTID") fields = ",".join(fields) if where is None: query = """SELECT {fields} from {tbl} """.format( tbl=self._table_name, fields=fields) else: query = """SELECT {fields} from {tbl} WHERE {where}""".format( tbl=self._table_name, fields=fields, where=where) if ftype is None: return pd.read_sql_query(query, self._con) elif str(ftype).lower() == 'esri': try: from arcgis.geometry import Geometry from arcgis.features import GeoAccessor, GeoSeriesAccessor df = pd.read_sql_query(query, self._con) fields = list(self.fields.keys()) lower_fields = [str(fld).lower() for fld in fields] if "shape" in lower_fields: idx = lower_fields.index("shape") SHAPE = fields[idx] df[SHAPE] = df[SHAPE].apply(lambda x: Geometry(x[8:])) df.spatial.set_geometry(SHAPE) try: df.spatial.project(self.wkid) except: print('nope') return df except ImportError: raise Exception( "The Python API for ArcGIS is required to import using ftype `esri`" ) except Exception as e: raise Exception(e)
def __setitem__(self, key, value): if value is None or \ (isinstance(value, str) and value == ""): self.data[key] = value else: value = Geometry(value) self.data[key] = value
def _from_xy(df, x_column, y_column, sr=None): """ """ from arcgis.geometry import SpatialReference, Geometry from arcgis.features import SpatialDataFrame if sr is None: sr = SpatialReference({'wkid': 4326}) if not isinstance(sr, SpatialReference): if isinstance(sr, dict): sr = SpatialReference(sr) elif isinstance(sr, int): sr = SpatialReference({'wkid': sr}) elif isinstance(sr, str): sr = SpatialReference({'wkt': sr}) geoms = [] for idx, row in df.iterrows(): geoms.append( Geometry({ 'x': row[x_column], 'y': row[y_column], 'spatialReference': sr })) df['SHAPE'] = geoms return SpatialDataFrame(data=df, sr=sr)
def _get_weighted_centroid_geometry(sub_df: pd.DataFrame, weighting_column: str, sptl_ref: SpatialReference, geom_col: str = 'SHAPE') -> tuple: """ Helper function to calculate centroid coordinates. Args: sub_df: DataFrame to calculate weighted coordinates for. weighting_column: Column to be used for weighting. sptl_ref: Spatial reference for the output geometry. geom_col: Optional - Geometry column, defaults to 'SHAPE'. Returns: Tuple of weighted centroid Geometry objects. """ # check if the weighting sum will be naught wgt_sum = sub_df[weighting_column].sum() # if there is a weighting sum, use it if wgt_sum > 0: wgt_x = np.average(sub_df[geom_col].apply(lambda geom: geom.centroid[0]), weights=sub_df[weighting_column]) wgt_y = np.average(sub_df[geom_col].apply(lambda geom: geom.centroid[1]), weights=sub_df[weighting_column]) # if there is not a weighting sum, just get the average centroid else: wgt_x = np.average(sub_df[geom_col].apply(lambda geom: geom.centroid[0])) wgt_y = np.average(sub_df[geom_col].apply(lambda geom: geom.centroid[1])) # create a point geometry at the weighted centroid wgt_geom = Geometry({'x': wgt_x, 'y': wgt_y, 'spatialReference': sptl_ref}) return wgt_geom
def weighted_polygon_centroid(poly_df, wgt_df, poly_id_fld='ID'): # create a spatial index for both spatially enabled dataframes to speed up the join if poly_df.spatial._sindex is None: poly_df.spatial.sindex() if wgt_df.spatial._sindex is None: wgt_df.spatial.sindex() # perform a spatial join between the points and the containing polygons join_df = wgt_df.spatial.join(poly_df) # extract the respective coordinates out of the geometry join_df['x'] = join_df['SHAPE'].apply(lambda geom: geom.centroid[0]) join_df['y'] = join_df['SHAPE'].apply(lambda geom: geom.centroid[1]) # get the id field following the join join_id_fld = poly_id_fld if poly_id_fld in join_df.columns else f'{poly_id_fld}_right' # calcuate the weighted centroid coordinates for each polygon and create geometries using these mean_df = join_df[[join_id_fld, 'x', 'y']].groupby(join_id_fld).mean() mean_df['SHAPE'] = mean_df.apply(lambda row: Geometry({ 'x': row['x'], 'y': row['y'], 'spatialReference': { 'wkid': 4326 } }), axis=1) # clean up the columns to a standard output schema mean_df.reset_index(inplace=True) mean_df = mean_df[[join_id_fld, 'SHAPE']].copy() mean_df.spatial.set_geometry('SHAPE') mean_df.columns = ['ID', 'SHAPE'] return mean_df
def _prep_sdf_for_nearest(input_dataframe: pd.DataFrame, id_column: str): """ Given an input Spatially Enabled Dataframe, prepare it to work well with the nearest solver. Args: input_dataframe: Spatially Enabled Dataframe with really any geometry. id_column: Field uniquely identifying each of location to be used for routing to nearest. Returns: Spatially Enabled Dataframe of points with correct columns for routing to nearest. """ # check inputs assert isinstance(input_dataframe, pd.DataFrame), f'The input dataframe must be a Pandas DataFrame, not ' \ f'{type(input_dataframe)}.' # ensure the geometry is set geom_col_lst = [ c for c in input_dataframe.columns if input_dataframe[c].dtype.name.lower() == 'geometry' ] assert len(geom_col_lst) > 0, 'The DataFrame does not appear to have a geometry column defined. This can be ' \ 'accomplished using the "df.spatial.set_geometry" method.' geom_col = geom_col_lst[0] # ensure the column is in the dataframe columns assert id_column in input_dataframe.columns, f'The provided id_column, "{id_column}," does not appear to be in ' \ f'the columns [{", ".join(input_dataframe.columns)}]"' # par down the input dataframe to just the columns needed input_dataframe = input_dataframe[[id_column, geom_col]].copy() # rename the columns to follow the schema needed for routing input_dataframe.columns = ['ID', 'SHAPE'] # ensure the spatial reference is WGS84 - if not, make it so if input_dataframe.spatial.sr.wkid != 4326: input_dataframe = input_dataframe.dm.project(4326) # if the geometry is not points, we still need points, so get the geometric centroids if input_dataframe.spatial.geometry_type != ['point']: input_dataframe['SHAPE'] = input_dataframe[geom_col].apply( lambda geom: Geometry({ 'x': geom.centroid[0], 'y': geom.centroid[1], 'spatialReference': geom.spatial_reference })) input_dataframe.spatial.set_geometry('SHAPE') # add a second column for the ID as Name input_dataframe['Name'] = input_dataframe['ID'] # ensure the geometry is correctly being recognized input_dataframe.spatial.set_geometry('SHAPE') # set the order of the columns and return return input_dataframe[['ID', 'Name', 'SHAPE']]
def _to_geo_array(values): from ._array import GeoArray, GeoType if isinstance(values, GeoArray): return values.data if not (isinstance(values, np.ndarray) and values.dtype == GeoType._record_type): values = [Geometry(v) for v in values] return np.asarray(values, dtype=GeoType._record_type)
def arcgis_to_pyprt(feature_set): """arcgis_to_pyprt(feature_set) -> List[InitialShape] This function allows converting an ArcGIS FeatureSet into a list of PyPRT InitialShape instances. You then typically call the ModelGenerator constructor with the return value if this function as parameter. Parameters: feature_set: FeatureSet Returns: List[InitialShape] """ initial_geometries = [] for feature in feature_set.features: try: geo = Geometry(feature.geometry) if geo.type == 'Polygon': coord = geo.coordinates()[0] coord_remove_last = coord[:-1] coord_inverse = np.flip(coord_remove_last, axis=0) if coord.shape[1] == 2: # we have to add a dimension coord_inverse[:, 1] *= -1 coord_add_dim = np.insert(coord_inverse, 1, 0, axis=1) coord_fin = np.reshape( coord_add_dim, (1, coord_add_dim.shape[0] * coord_add_dim.shape[1])) elif coord.shape[1] == 3: # need to swap the 1 and 2 columns coord_inverse[:, 1] *= -1 coord_swap_dim = coord_inverse.copy() temp = np.copy(coord_swap_dim[:, 1]) coord_swap_dim[:, 1] = coord_swap_dim[:, 2] coord_swap_dim[:, 2] = temp coord_fin = np.reshape( coord_swap_dim, (1, coord_swap_dim.shape[0] * coord_swap_dim.shape[1])) initial_geometry = pyprt.InitialShape(coord_fin.tolist()[0]) initial_geometries.append(initial_geometry) except: print("This feature is not valid: ") print(feature) print() return initial_geometries
def to_wkt(source): if source is None: return None if isinstance(source, Geometry): return source.WKT if isinstance(source, arcpy.Geometry): return source.WKT geom = Geometry(source) return geom.WKT
def _as_arcgis(self, spatial_reference): """Return an arcgis Geometry. Arguments: spatial_reference A spatial reference integer code or definition dictionary, for example {'wkid': 3857} """ if isinstance(spatial_reference, int): spatial_reference = {'wkid': spatial_reference} if isinstance(self, ShapelyPoint) or \ isinstance(self, ShapelyMultiPoint) or \ isinstance(self, ShapelyLineString) or \ isinstance(self, ShapelyMultiLineString): geom = Geometry(self.__geo_interface__) geom['spatialReference'] = spatial_reference elif isinstance(self, ShapelyPolygon): linear_rings = [self.exterior] linear_rings += self.interiors rings = [ list(r.__geo_interface__['coordinates']) for r in linear_rings ] geom = Geometry({ 'rings': rings, 'spatialReference': spatial_reference }) elif isinstance(self, ShapelyMultiPolygon): linear_rings = [] for poly in self.geoms: linear_rings += [poly.exterior] linear_rings += poly.interiors rings = [ list(r.__geo_interface__['coordinates']) for r in linear_rings ] geom = Geometry({ 'rings': rings, 'spatialReference': spatial_reference }) return geom
def new_dest(): geom = Geometry({ 'x': -122.7342423, 'y': 45.4383307, 'spatialReference': { 'latestWkid': 4326, 'wkid': 4326 } }) return geom
def validate_img_gis(geo_gis, img_url): try: img_lyr = ImageryLayer(img_url, gis=geo_gis) img_lyr.get_samples(Geometry({ "x": -122.435, "y": 37.785 }), geometry_type='esriGeometryPoint') print('Thematic Service GIS: {}'.format(geo_gis)) except RuntimeError: raise Exception('{} Does Not Support getSamples on Service: {}'.format( geo_gis, img_url))
def from_featureset(fset, sr=None): """ Converts a FeatureSet to a pd.DataFrame =============== ============================================== Arguments Description --------------- ---------------------------------------------- fset Required FeatureSet. FeatureSet object. =============== ============================================== return Panda's DataFrame """ if isinstance(fset, FeatureSet): rows = [] sr = fset.spatial_reference dt_fields = [ fld['name'] for fld in fset.fields if fld['type'] == 'esriFieldTypeDate' ] if sr is None: sr = {'wkid': 4326} for feat in fset.features: a = feat.attributes if not feat.geometry is None: g = feat.geometry g['spatialReference'] = sr a['SHAPE'] = Geometry(g) rows.append(a) del a, feat from arcgis.features import GeoAccessor, GeoSeriesAccessor df = pd.DataFrame(data=rows) for fld in dt_fields: try: df[fld] = pd.to_datetime(df[fld] / 1000, infer_datetime_format=True, unit='s') except: df[fld] = pd.to_datetime(df[fld], infer_datetime_format=True) if 'SHAPE' in df.columns: df.spatial.set_geometry("SHAPE") df.spatial.sr = sr return df else: return None
def get_esri_geometry_for_h3_id(h3_id: str) -> Polygon: """ Convert an Uber H3 id to an ArcGIS Python API Polygon Geometry object. :param h3_id:String Uber H3 id. :return: ArcGIS Python API Polygon Geometry object """ # get a list of coordinate rings for the hex id's coord_lst = [h3.h3_to_geo_boundary(h3_id, geo_json=True)] # creat a geometry object using this geometry list return Geometry({ "type": "Polygon", "coordinates": coord_lst, 'spatialReference': { 'wkid': 4326 } })
def soil_type(self): parcel_geometry = wkt.loads(self.transformed.wkt) geometry_map = mapping(parcel_geometry) esriPolygon = { "spatialReference": { "wkid": 31370 }, "rings": [[list(p) for p in r] for r in geometry_map["coordinates"]] } response = requests.get( f"https://geoservices.wallonie.be/arcgis/rest/services/SOL_SOUS_SOL/CNSW__PRINC_TYPES_SOLS/MapServer/identify?f=json&geometry={esriPolygon}&tolerance=0&mapExtent={parcel_geometry.bounds}&sr=31370&imageDisplay=1,1,1&layers=all&geometryType=esriGeometryPolygon&geometryPrecision=4" ) if not response.ok: raise 'Could not define the soil type' if "error" in response.json().keys(): raise response.json() soil_type_data = {} for data in response.json()["results"]: geometry = wkt.loads(Geometry(data["geometry"]).WKT) code = data["attributes"]["CODE"] soil_type_data[code] = { "code": code, "description": data["attributes"]["DESCRIPTION"], "area": 0 } geometry = cascaded_union(geometry).buffer(0) intersection = geometry.intersection(parcel_geometry) soil_type_data[code]["area"] += intersection.area for val in soil_type_data.values(): val["proportion"] = val["area"] / parcel_geometry.area return soil_type_data.values()
def prep_sdf_for_nearest(df, id_fld, weighting_points=None): """ Given an input Spatially Enabled Dataframe, prepare it to work well with the nearest solver. :param df: Spatially Enabled Dataframe with really any geometry. :param id_fld: Field uniquely identifying each of location to be used for routing to nearest. :param weighting_points: Spatially Enabled Dataframe of points for calculating weighted centroids. :return: Spatially Enabled Dataframe of points with correct columns for routing to nearest. """ # par down the input dataframe to just the columns needed df = df[[id_fld, 'SHAPE']].copy() # rename the columns to follow the schema needed for routing df.columns = ['ID', 'SHAPE'] # if the geometry is polygons and there is a weighting points spatially enabled dataframe if df.spatial.geometry_type == ['polygon' ] and weighting_points is not None: # calculate the weighted centroid for the polygons df = weighted_polygon_centroid(df, weighting_points) # otherwise, if the geometry is not points, we still need points, so just get the geometric centroids # TODO: Account for polygons NOT always being in WGS 84 elif df.spatial.geometry_type != ['point'] and weighting_points is None: df['SHAPE'] = df['SHAPE'].apply( lambda geom: Geometry({ 'x': geom.centroid[0], 'y': geom.centroid[1], 'spatialReference': { 'wkid': 4326 } })) # add a second column for the ID as Name df['Name'] = df['ID'] # ensure the geometry is correctly being recognized df.spatial.set_geometry('SHAPE') # set the order of the columns and return return df[['ID', 'Name', 'SHAPE']].copy()
def set_features(self): df_list = [] for idx, row in enumerate( self.grid_sdf.iterrows()): #self.grid_sdf.iterrows()): geom = Geometry(row[1].SHAPE) print(idx) sp_filter = filters.intersects(geom, self.grid_wkid) data_fl = FeatureLayer(url=self.feat_url, gis=self.gis_conn) ## Change return all records to True df_list.append( data_fl.query(geometry_filter=sp_filter, return_all_records=self.return_all_records).df) self.features = df_list
def _convert_geometry_column_to_geometry(df, geometry_column): """ Helper function to follow the paradigm of DRYD (don't repeat yourself, DUMMY!) for converting a column from object (str) to a valid Geometry type, and enabling this column as the recognized geometry column for a fully functional Spatially Enabled Dataframe. Args: df: Pandas DataFrame with a column containing valid Esri JSON objects describing Geometries. geometry_column: Column containing the valid Esri Geometry object instances. Returns: Spatially Enabled Pandas DataFrame """ # convert the values in the geometry column to geometry objects df[geometry_column] = df[geometry_column].apply( lambda val: Geometry(json.loads(val))) # tell the GeoAccessor to recognize the column df.spatial.set_geometry(geometry_column) return df
def split_at_point(self, point_geometry): """ Returns two polyline geometry objects as a list split at the intersection of the line. :param point_geometry: Required arcgis.geometry.Point ArcGIS Point geometry defining the location the line will be split at. :return: Two item list of arcgis.geometry.Polyline objects List of two ArcGIS Polyline objects, one on either side of the input point location. """ if not isinstance(self, Polyline): raise Exception('Split at point can only be performed on a Polyline geometry object.') if not isinstance(point_geometry, Point): raise Exception('Split at point requires a Point geometry object to define the split location.') if self.spatial_reference is None: raise Warning('The spatial reference for the line to be split is not defined.') if point_geometry.spatial_reference is None: raise Warning('The spatial reference of the point defining the split location is not defined.') if (self.spatial_reference != point_geometry.spatial_reference and self.spatial_reference.wkid != point_geometry.spatial_reference.wkid and self.spatial_reference.latestWkid != point_geometry.spatial_reference.wkid and self.spatial_reference.wkid != point_geometry.spatial_reference.latestWkid and self.spatial_reference.latestWkid != point_geometry.spatial_reference.latestWkid): raise Exception('The spatial reference for the line and point are not the same.') # if HASARCPY: # raise Exception('Not yet implemented') if HASSHAPELY: linestring_geometry = self.as_shapely point_geometry = point_geometry.as_shapely split_result = ops.split(linestring_geometry, point_geometry) polyline_list = [Geometry({ 'paths': [line_string.__geo_interface__['coordinates']], 'spatialReference': self.spatial_reference}) for line_string in split_result] return polyline_list else: raise Exception('Shapely is required to perform split_at_point')
def _from_xy(df, x_column, y_column, sr=None): """ Takes an X/Y Column and Creates a Point Geometry from it. """ from arcgis.geometry import SpatialReference, Geometry if sr is None: sr = SpatialReference({'wkid' : 4326}) if not isinstance(sr, SpatialReference): if isinstance(sr, dict): sr = SpatialReference(sr) elif isinstance(sr, int): sr = SpatialReference({'wkid' : sr}) elif isinstance(sr, str): sr = SpatialReference({'wkt' : sr}) geoms = [] for idx, row in df.iterrows(): geoms.append( Geometry({'x' : row[x_column], 'y' : row[y_column], 'spatialReference' : sr}) ) df['SHAPE'] = geoms df.spatial.set_geometry('SHAPE') return df
def set_selected(self, indicator): created = False out_sdf = None # Indicator Feature Layer indicator_url = self.__getattribute__(indicator + '_url') print(indicator_url) #data_fl = FeatureLayer(url=indicator_url, gis=self.gis_conn) # Enumerate Used to Leverage the Merge Method on the Data Frame. # Set the First and Merge the Remainder to the First. for idx, row in enumerate(self.grid_sdf.iterrows()): # Negative Buffer Used to Avoid Selecting More Than 1 Cell sp_filter = filters.intersects( Geometry(row[1].SHAPE).buffer(-.1), self.grid_wkid) if not indicator_url: df_current = SpatialDataFrame( columns=field_schema.get(indicator), geometry=[Geometry(json.loads(row[1].SHAPE.JSON))]) created = True else: # Negative Buffer to Select a Single Grid Cell sp_filter = filters.intersects( Geometry(row[1].SHAPE).buffer(-.1), self.grid_wkid) df_current = data_fl.query( geometry_filter=sp_filter, return_all_records=self.return_all_records).df # Set The First Instance if idx == 0: # Check If Cell Found in Target Indicator Layer if df_current.empty: # Use Grid SDF Row Geom to Insert Empty Record for Target Indicator data_fl.edit_features( adds=[{ 'attributes': {}, 'geometry': json.loads(row[1].SHAPE.JSON) }]) # Select Newly Created Cell As Input out_sdf = data_fl.query( geometry_filter=sp_filter, return_all_records=self.return_all_records).df else: # Use Matched Grid Cell out_sdf = df_current # Append Additional Data else: # Check If Cell Found in Target Indicator Layer if df_current.empty: # Use Grid SDF Row Geom to Insert Empty Record for Target Indicator data_fl.edit_features( adds=[{ 'attributes': {}, 'geometry': json.loads(row[1].SHAPE.JSON) }]) # Select Newly Created Cell As Input out_sdf.merge( data_fl.query( geometry_filter=sp_filter, return_all_records=self.return_all_records).df) else: # Use Matched Grid Cell out_sdf = out_sdf.merge_datasets(df_current) self.selected = out_sdf.reset_index(drop=False) print("Selected: " + str(len(out_sdf))) return created
class AccessPutin(unittest.TestCase): canyon_reach_id = 3066 putin_point = Geometry({ 'x': -121.633094, 'y': 45.79532367, 'spatialReference': { 'wkid': 4326 } }) point_type = 'access' subtype = 'putin' name = 'Hayes Creek' side_of_river = 'left' collection_method = 'digitized' collection_date = '02 Nov 1998' test_series = pd.Series({ "_geometry": Geometry({ 'x': -121.633094, 'y': 45.79532367, 'spatialReference': { 'wkid': 4326 } }), "reach_id": str(canyon_reach_id), "point_type": point_type, "subtype": subtype, "name": name, "side_of_river": side_of_river, "nhdplus_measure": None, "nhdplus_reach_id": None, "collection_method": collection_method, "collection_date": collection_date, "notes": None, "description": None, "type": point_type }) test_feature_dict = { "geometry": { "x": -121.633094, 'y': 45.79532367, "spatialReference": { "wkid": 4326 } }, "attributes": { 'reach_id': '3066', 'point_type': 'access', 'subtype': 'putin', 'name': 'Hayes Creek', 'side_of_river': 'left', 'nhdplus_measure': None, 'nhdplus_reach_id': None, 'collection_method': 'digitized', 'update_date': '02 Nov 1998', 'notes': None, 'description': None, 'difficulty': None } } test_snap_geom_dict = { 'x': -121.63309439504, 'y': 45.7953235252763, 'spatialReference': { 'wkid': 4326 } } def test_instantiate_access(self): access = ReachPoint(reach_id=self.canyon_reach_id, geometry=self.putin_point, point_type=self.point_type, subtype=self.subtype, name=self.name, side_of_river=self.side_of_river, collection_method=self.collection_method, update_date=self.collection_date) self.assertEqual(type(access), ReachPoint) def test_as_feature(self): access = ReachPoint(reach_id=self.canyon_reach_id, geometry=self.putin_point, point_type=self.point_type, subtype=self.subtype, name=self.name, side_of_river=self.side_of_river, collection_method=self.collection_method, update_date=self.collection_date) delattr( access, 'uid') # since this will be different every time, just remove it self.assertDictEqual(access.as_feature.as_dict, self.test_feature_dict) def test_snap_geom_to_nhdplus(self): access = ReachPoint(reach_id=self.canyon_reach_id, geometry=self.putin_point, point_type=self.point_type, subtype=self.subtype, name=self.name, side_of_river=self.side_of_river, collection_method=self.collection_method, update_date=self.collection_date) access.snap_to_nhdplus() self.assertDictEqual(access.as_feature.as_dict['geometry'], self.test_snap_geom_dict)
def process_by_metadata(gis): return_all_records = False look_back_days = config.look_back_days dates = csl.get_dates_in_range(look_back_days) where_clause = csl.form_query_string(dates) grid_fl = FeatureLayer(url=config.grid_url) grid_sdf = grid_fl.query(return_all_records=return_all_records, where=where_clause).df geometry = grid_sdf.geometry sr = {'wkid': 4326} sp_rel = "esriSpatialRelIntersects" for idx, row in enumerate(grid_sdf.iterrows()): geom = row[1].SHAPE new_geom = Geometry({ "rings": [[[geom.extent.upperRight.X - .1, geom.extent.lowerLeft.Y + .1], [geom.extent.lowerLeft.X + .1, geom.extent.lowerLeft.Y + .1], [geom.extent.lowerLeft.X + .1, geom.extent.upperRight.Y - .1], [geom.extent.upperRight.X - .1, geom.extent.upperRight.Y - .1], [geom.extent.upperRight.X - .1, geom.extent.lowerLeft.Y + .1]]], "spatialReference": { "wkid": 4326 } }) grid_filter = filters._filter(new_geom, sr, sp_rel) sp_filter = filters._filter(geom, sr, sp_rel) data_fl = FeatureLayer(url=config.features_url) #out_fields=in_fields, data_sdf = data_fl.query(geometry_filter=sp_filter, return_geometry=True, return_all_records=return_all_records).df print('Processing Completeness') #bounding_box = '(37.708132, -122.513617, 37.832132, -122.349607)' bounding_box = '(' + \ str(geom.extent.lowerLeft.Y) + ',' + \ str(geom.extent.lowerLeft.X) + ',' + \ str(geom.extent.upperRight.Y) + ',' + \ str(geom.extent.upperRight.X) + ')' osm_sdf = runner.gen_osm_sdf('line', bounding_box, osm_tag='highway', present=True) completeness_sdf, completeness_fl = comp.completeness( gis, osm_sdf, data_sdf, config.completeness_url, grid_filter, geom) print(completeness_sdf) #update_features(them_acc_sdf, them_acc_fl) print('Completeness Updated') print('Processing Logical Consistency') lc_sdf, lc_fl = lc.logical_consisitency( gis, config.template_fc, config.template_gdb, config.attr_check_file, config.attr_check_tab, data_sdf, config.features_url, config.logical_consistency_url, grid_filter, geom, config.attr_error_field_count, config.attr_error_field_def) print(lc_sdf) update_features(lc_sdf, lc_fl) print('Logical Consistency Updated.') print('Processing temporal currency') tc_sdf, tc_fl = tc.temporal_currency(gis, data_sdf, config.currency_url, grid_filter, geom, config.currency_field) print(tc_sdf) #update_features(tc_sdf, tc_fl) print('Temporal Currency Updated') print('Processing source lineage') sl_sdf, sl_fl = sl.source_lineage(gis, data_sdf, config.source_lineage_url, grid_filter, geom, config.search_field, config.value_field) print(sl_sdf) #update_features(sl_sdf, sl_fl) print('Source Lineage Updated') print('Processing Positional Accuracy') pa_sdf, pa_fl = pa.positional_accuracy(gis, data_sdf, config.positional_acc_url, grid_filter, geom, config.positional_acc_field) print(pa_sdf) #update_features(pa_sdf, pa_fl) print('Positional Accuracy Updated') print('Processing Thematic Accuracy') them_acc_sdf, them_acc_fl = them_acc.thematic_accuracy( gis, data_sdf, config.thematic_url, grid_filter, geom, config.thematic_acc_field) print(them_acc_sdf) #update_features(them_acc_sdf, them_acc_fl) print('Theamatic Accuracy Updated') return
col='ST') #column to get unique values from # In[11]: item = gis.content.get("8444e275037549c1acab02d2626daaee") flayer = item.layers[0] df2 = flayer.query().sdf # In[12]: fset = flayer.query() # In[13]: from arcgis.geometry import Geometry g = Geometry(fset.features[0].geometry) g.as_arcpy # In[14]: #create an opacity visual variable based on a percent of dominant parties in registered citizens. opacity_expression = ( "var republican = $feature.MP06025a_B;var democrat = $feature.MP06024a_B;" "var independent = $feature.MP06026a_B;var parties = [republican, democrat, independent];" "var total = Sum(parties);var max = Max(parties);return (max / total) * 100;" ) opacity_stops = [{ "value": 33, "transparency": 0.05 * 255, "label": "< 33%" }, {
def get_dataframe(in_features, gis=None): """ Get a spatially enabled dataframe from the input features provided. :param in_features: Spatially Enabled Dataframe | String path to Feature Class | pathlib.Path object to feature class | ArcGIS Layer object |String url to Feature Service | String Web GIS Item ID Resource to be evaluated and converted to a Spatially Enabled Dataframe. :param gis: Optional GIS object instance for connecting to resources. """ # if a path object, convert to a string for following steps to work correctly in_features = str(in_features) if isinstance(in_features, pathlib.Path) else in_features # helper for determining if feature layer def _is_feature_layer(in_ftrs): if hasattr(in_ftrs, 'isFeatureLayer'): return in_ftrs.isFeatureLayer else: return False # if already a Spatially Enabled Dataframe, mostly just pass it straight through if isinstance(in_features, pd.DataFrame) and in_features.spatial.validate() is True: df = in_features # if a csv previously exported from a Spatially Enabled Dataframe, get it in elif isinstance(in_features, str) and os.path.exists(in_features) and in_features.endswith('.csv'): df = pd.read_csv(in_features) df['SHAPE'] = df['SHAPE'].apply(lambda geom: Geometry(eval(geom))) # this almost always is the index written to the csv, so taking care of this if df.columns[0] == 'Unnamed: 0': df = df.set_index('Unnamed: 0') del (df.index.name) # create a Spatially Enabled Dataframe from the direct url to the Feature Service elif isinstance(in_features, str) and in_features.startswith('http'): # submitted urls can be lacking a few essential pieces, so handle some contingencies with some regex matching regex = re.compile(r'((^https?://.*?)(/\d{1,3})?)\?') srch = regex.search(in_features) # if the layer index is included, still clean by dropping any possible trailing url parameters if srch.group(3): in_features = f'{srch.group(1)}' # ensure at least the first layer is being referenced if the index was forgotten else: in_features = f'{srch.group(2)}/0' # if the layer is unsecured, a gis is not needed, but we have to handle differently if gis is not None: df = FeatureLayer(in_features, gis).query(out_sr=4326, as_df=True) else: df = FeatureLayer(in_features).query(out_sr=4326, as_df=True) # create a Spatially Enabled Dataframe from a Web GIS Item ID elif isinstance(in_features, str) and len(in_features) == 32: # if publicly shared on ArcGIS Online this anonymous gis can be used to access the resource if gis is None: gis = GIS() itm = gis.content.get(in_features) df = itm.layers[0].query(out_sr=4326, as_df=True) elif isinstance(in_features, (str, pathlib.Path)): df = GeoAccessor.from_featureclass(in_features) # create a Spatially Enabled Dataframe from a Layer elif _is_feature_layer(in_features): df = FeatureSet.from_json(arcpy.FeatureSet(in_features).JSON).sdf # sometimes there is an issue with modified or sliced dataframes with the SHAPE column not being correctly # recognized as a geometry column, so try to set it as the geometry...just in case elif isinstance(in_features, pd.DataFrame) and 'SHAPE' in in_features.columns: in_features.spatial.set_geometry('SHAPE') df = in_features if df.spatial.validate() is False: raise Exception('Could not process input features for get_dataframe function. Although the input_features ' 'appear to be in a Pandas Dataframe, the SHAPE column appears to not contain valid ' 'geometries. The Dataframe is not validating using the *.spatial.validate function.') else: raise Exception('Could not process input features for get_dataframe function.') # ensure the universal spatial column is correctly being recognized df.spatial.set_geometry('SHAPE') return df
def thematic_accuracy(out_sdf, df_list, f_thm_acc, them_gis, them_url): print('Running Thematic Accuracy') f_thm_acc = validate_field(f_thm_acc, list(df_list[0])) # List Used for Logging Differences in Population Sources pop_diff = [] for idx, row in enumerate(out_sdf.iterrows()): df_current = df_list[idx] ##----------------------------------------------------------------------------- ## Uses Geoenrichment - Not available outside of AGOL # Pull GeoEnrichment Figures # enriched = enrich([row[1]['SHAPE']], gis=geo_gis) # if 'TOTPOP' not in list(enriched): # enriched_pop = -1 # else: # enriched_pop = enriched.TOTPOP[0] # # # Pull Samples From Configured Population Service # img_lyr = ImageryLayer(img_url, gis=geo_gis) # cells = img_lyr.properties.maxImageHeight * img_lyr.properties.maxImageWidth # samples = img_lyr.get_samples( # row[1]['SHAPE'], # geometry_type='esriGeometryPolygon', # sample_count=cells # ) # sample_total = sum([int(sample['value']) for sample in samples]) # # # Push Significant Values Into List for Averaging # if enriched_pop or sample_total < 100: # pass # else: # diff = abs(enriched_pop - sample_total) # if diff > 100: # pop_diff.append(diff) # # tot_pop = enriched_pop if enriched_pop > 0 else sample_total # tot_pop = tot_pop if tot_pop > 0 else -1 ##----------------------------------------------------------------------------- them_lyr = FeatureLayer(url=them_url, gis=them_gis) geom = Geometry(row[1].SHAPE).buffer(-.01) sp_filter = filters.intersects(geom, 4326) them_sdf = them_lyr.query(geometry_filter=sp_filter, return_all_records=True).df #print(them_sdf) if len(df_current) > 0: count = len(df_current) max_val = df_current[f_thm_acc].max() max_scale = 100 * ( len(df_current[df_current[f_thm_acc] == max_val]) / count) min_val = df_current[f_thm_acc].min() min_scale = 100 * ( len(df_current[df_current[f_thm_acc] == min_val]) / count) vc = df_current[f_thm_acc].value_counts() common = df_current[f_thm_acc].mode() # Used in MSP mean = df_current[f_thm_acc].mean() if len(common) > 0: common = common[0] common_count = vc[common] common_per = (vc[common] / count) * 100 else: common = min_val common_count = 1 common_per = 100 count_2500 = 0 count_5000 = 0 count_12500 = 0 count_25000 = 0 count_50000 = 0 count_100000 = 0 count_250000 = 0 count_500000 = 0 count_1000000 = 0 if 2500 in vc: count_2500 = vc[2500] if 5000 in vc: count_5000 = vc[5000] if 12500 in vc: count_12500 = vc[12500] if 25000 in vc: count_25000 = vc[25000] if 50000 in vc: count_50000 = vc[50000] if 100000 in vc: count_100000 = vc[100000] if 250000 in vc: count_250000 = vc[250000] if 500000 in vc: count_500000 = vc[500000] if 1000000 in vc: count_1000000 = vc[1000000] MSP = get_msp(scale=common) # SHOULD UPDATE MISSION_PLANNING FIELD if not out_sdf['MEAN'][0]: m = 0 else: m = out_sdf['MEAN'][0] SCORE_VALUE = them_sdf['grls_score'].loc[ 0] #get_equal_breaks_score(m)# get_equal_breaks_score(output_features, ['MEAN','EQUAL']) # PUT SCORE IN EQUAL #GRLS = SCORE_VALUE #domScale = common # FIELD 1 is the source, Field 2 is the field to be updated #df_current['EQUAL'] = SCORE_VALUE # ASSIGNS EQUAL TO LANSCAN_SCALE #29 field out_sdf.set_value(idx, field_schema.get('them')[0], common) # median out_sdf.set_value(idx, field_schema.get('them')[1], common_count) # % common out_sdf.set_value(idx, field_schema.get('them')[2], round(common_per, 1)) out_sdf.set_value(idx, field_schema.get('them')[3], min_val) out_sdf.set_value(idx, field_schema.get('them')[4], round(min_scale, 1)) out_sdf.set_value(idx, field_schema.get('them')[5], max_val) out_sdf.set_value(idx, field_schema.get('them')[6], round(max_scale, 1)) out_sdf.set_value(idx, field_schema.get('them')[7], count_2500) out_sdf.set_value(idx, field_schema.get('them')[8], count_5000) out_sdf.set_value(idx, field_schema.get('them')[9], count_12500) out_sdf.set_value(idx, field_schema.get('them')[10], count_25000) out_sdf.set_value(idx, field_schema.get('them')[11], count_50000) out_sdf.set_value(idx, field_schema.get('them')[12], count_100000) out_sdf.set_value(idx, field_schema.get('them')[13], count_250000) out_sdf.set_value(idx, field_schema.get('them')[14], count_500000) out_sdf.set_value(idx, field_schema.get('them')[15], count_1000000) out_sdf.set_value(idx, field_schema.get('them')[16], round(count_2500 * 100 / count, 1)) out_sdf.set_value(idx, field_schema.get('them')[17], round(count_5000 * 100 / count, 1)) out_sdf.set_value(idx, field_schema.get('them')[18], round(count_12500 * 100 / count, 1)) out_sdf.set_value(idx, field_schema.get('them')[19], round(count_25000 * 100 / count, 1)) out_sdf.set_value(idx, field_schema.get('them')[20], round(count_50000 * 100 / count, 1)) out_sdf.set_value(idx, field_schema.get('them')[21], round(count_100000 * 100 / count, 1)) out_sdf.set_value(idx, field_schema.get('them')[22], round(count_250000 * 100 / count, 1)) out_sdf.set_value(idx, field_schema.get('them')[23], round(count_500000 * 100 / count, 1)) out_sdf.set_value(idx, field_schema.get('them')[24], round(count_1000000 * 100 / count, 1)) out_sdf.set_value(idx, field_schema.get('them')[25], count) out_sdf.set_value(idx, field_schema.get('them')[26], str(MSP)) #MISSION_PLANNING FIELD out_sdf.set_value(idx, field_schema.get('them')[27], SCORE_VALUE) #), # THEMATIC SCALE VALUE #out_sdf.set_value(idx, field_schema.get('them')[27], tot_pop) # ), # THEMATIC SCALE VALUE out_sdf.set_value(idx, field_schema.get('them')[28], population_scale( common, SCORE_VALUE)) # POPULATION_SCALE out_sdf.set_value(idx, field_schema.get('them')[29], mean) #to 28 else: for i in range(0, 25): out_sdf.set_value(idx, field_schema.get('them')[i], -1) out_sdf.set_value(idx, field_schema.get('them')[25], 0) out_sdf.set_value(idx, field_schema.get('them')[26], 'N/A') out_sdf.set_value(idx, field_schema.get('them')[27], 'N/A') out_sdf.set_value(idx, field_schema.get('them')[28], 0) out_sdf.set_value(idx, field_schema.get('them')[29], -1) del df_current #print('Average Difference of Population Estimates: {}'.format(np.average(pop_diff))) return out_sdf
def completeness(out_sdf, df_list, osm_sdf): print('Running Completeness') for idx, row in enumerate(out_sdf.iterrows()): before_val = None geom = Geometry(row[1].SHAPE) # Unpack Geom Extent as OSM Expects bbox = (geom.extent[1], geom.extent[0], geom.extent[3], geom.extent[2]) # Fetch OSM SpatialDataFrame osm_sdf = gen_osm_sdf('line', bbox, osm_tag='highway') data_sdf = df_list[idx] if len(data_sdf) == 0: before_val = 0 else: sq = data_sdf[data_sdf.geometry.notnull()].geometry.disjoint( geom) == False df_before = data_sdf[sq].copy() geoms_before = df_before.clip(geom.extent) geoms_before_sdf = SpatialDataFrame(geometry=geoms_before) q_before = geoms_before_sdf['SHAPE'] == {"paths": []} geoms_before_sdf = geoms_before_sdf[~q_before].copy() geoms_before_sdf.reset_index(inplace=True, drop=True) geometry_type = osm_sdf.geometry_type sq = osm_sdf[osm_sdf.geometry.notnull()].geometry.disjoint( geom) == False df_after = osm_sdf[sq].copy() geoms_after = df_after.clip(geom.extent) geoms_after_sdf = SpatialDataFrame(geometry=geoms_after) #geoms_after_sdf = SpatialDataFrame({'Pass': '******'}, geometry=geoms_after, index=[0]) q_after = geoms_after_sdf['SHAPE'] == {"paths": []} geoms_after_sdf = geoms_after_sdf[~q_after].copy() geoms_after_sdf.reset_index(inplace=True, drop=True) # This Need Work if geometry_type == "Polygon": if before_val == None: before_val = geoms_before_sdf.geometry.project_as( 4326).get_area('GEODESIC', 'SQUAREKILOMETERS').sum() after_val = geoms_after_sdf.geometry.project_as(4326).get_area( 'GEODESIC', 'SQUAREKILOMETERS').sum() if after_val > 0: score = get_cp_score(ratio=before_val / after_val, baseVal=before_val, inputVal=after_val) else: score = get_cp_score(0, before_val, after_val) out_sdf.set_value(idx, field_schema.get('cmpl')[0], round(before_val, 1)) out_sdf.set_value(idx, field_schema.get('cmpl')[1], round(after_val, 1)) out_sdf.set_value(idx, field_schema.get('cmpl')[3], round(before_val - after_val, 1)) out_sdf.set_value(idx, field_schema.get('cmpl')[2], score) elif geometry_type == "Polyline": if before_val == None: geom = geoms_before_sdf.geometry geom_projected = geoms_before_sdf.geometry.project_as(3857) before_val = int(sum(geom_projected.length.tolist())) geom_projected = geoms_before_sdf.geometry.project_as(3857) after_val = int(sum(geom_projected.length.tolist())) if after_val > 0: score = get_cp_score(ratio=before_val / after_val, baseVal=before_val, inputVal=after_val) else: score = get_cp_score(0, before_val, after_val) out_sdf.set_value(idx, field_schema.get('cmpl')[0], round(before_val, 1)) out_sdf.set_value(idx, field_schema.get('cmpl')[1], round(after_val, 1)) out_sdf.set_value(idx, field_schema.get('cmpl')[3], round(before_val - after_val, 1)) out_sdf.set_value(idx, field_schema.get('cmpl')[2], score) else: if before_val == None: before_count = len(geoms_before_sdf) else: before_count = 0 after_count = len(geoms_after_sdf) if after_count > 0: score = get_cp_score(ratio=before_count / after_count, baseVal=before_count, inputVal=after_count) else: score = get_cp_score(ratio=0, baseVal=before_count, inputVal=after_count) out_sdf.set_value(idx, field_schema.get('cmpl')[0], before_count) out_sdf.set_value(idx, field_schema.get('cmpl')[1], after_count) out_sdf.set_value(idx, field_schema.get('cmpl')[3], before_count - after_count) out_sdf.set_value(idx, field_schema.get('cmpl')[2], score) del sq del df_after del geom if before_val != None: print(before_val) # del df_before return out_sdf
x, y = p if x >= m1 and x <= m2 and y >= n1 and y <= n2: return True return False return (point_in((x1, y1), b)) or (point_in((x2, y1), b)) or (point_in((x2, y2), b)) or (point_in((x1, y2), b)) # iterate through image tiles on naip_image_layer starting from bottom row for y_idx in range(y_num_tile): for x_idx in range(x_num_tile): image_name = str(y_idx*x_num_tile + x_idx) x_start = min_x + x_idx * tile_size y_start = min_y + y_idx * tile_size # export annotations for buildings in the image tile_image_geometry = Geometry({ "rings" : [[[x_start,y_start],[x_start+tile_size,y_start],[x_start+tile_size,y_start+tile_size],[x_start,y_start+tile_size]]], "spatialReference" : {"wkid" : crs_id} }) annotations = [] for idx, bbox in enumerate(building_data.geometry): # bbox is polygon bbox_extent = bbox.extent tile_extent = tile_image_geometry.extent try: # if bbox overlaps with tile image extent, record the building bbox if overlap(bbox_extent, tile_extent): # bbox contains normalized [xywh] x1_r, y1_r, x2_r, y2_r = bbox_extent # clipping x1 = max(x_start, x1_r) y1 = max(y_start, y1_r) x2 = min(x2_r, x_start+tile_size)