def to_df(self) -> pd.DataFrame:
# convert the layer to a spatially enabled dataframe
df = GeoAccessor.from_featureclass(self)
# if the objectid field is in the columns, remove it
oid_col_lst = [col for col in df.columns if col.lower == 'objectid']
if len(oid_col_lst) > 0:
df.drop(oid_col_lst[0], axis=1, inplace=True)
# return the dataframe
return df
def process_cc_poi(input_fc, output_fc):
""" Take the old points_of_interest feature class as input.
Clean it.
Write a feature class in local projection.
Returns the name of the feature class.
Uses the workspace as the output location. """
# For some reason the CC poi data was stored in Web Mercator.
sdf = GeoAccessor.from_featureclass(input_fc)
print(sdf.head(5))
print("Rows:", len(sdf))
print("Columns: ", sdf.columns)
print("Types:", sdf.dtypes)
# Clean!
# Columns that we're keeping
# There are a bunch ignored including COUNTY STATE COUNTRY
# and specialize ones that probably only matter to GeoComm like rSrcId
# There are two GlobalId fields, huh. That's annoying. I keep only 1.
sdf = sdf[[
'SHAPE',
'Name',
'Type',
"Type2",
'Creator',
'Created',
"Editor",
"Last_Edit",
]]
# Rename
sdf.rename(columns={
'Name': 'name',
'Type': 'category',
'Type2': 'subcategory',
'Creator': 'created_user',
'Created': 'created_date',
'Editor': 'last_edited_user',
'Last_Edit': 'last_edited_date',
},
inplace=True)
# Change column types
sdf[['created_date']] = sdf[['created_date']].astype("datetime64")
sdf[['last_edited_date']] = sdf[['last_edited_date']].astype("datetime64")
# Add some fields and set defaults on them.
sdf.loc[:, 'source'] = 'clatsop'
print("We have %d points." % len(sdf))
print(sdf.head(5))
print(sdf.columns)
print(sdf.dtypes)
wm_fc = output_fc + '_wm'
sdf.spatial.to_featureclass(wm_fc, sanitize_columns=False)
print("Wrote %d points to \"%s\"." % (len(sdf), wm_fc))
sref = """PROJCS['NAD_1983_HARN_StatePlane_Oregon_North_FIPS_3601_Feet_Intl',GEOGCS['GCS_North_American_1983_HARN',DATUM['D_North_American_1983_HARN',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Lambert_Conformal_Conic'],PARAMETER['False_Easting',8202099.737532808],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',-120.5],PARAMETER['Standard_Parallel_1',44.33333333333334],PARAMETER['Standard_Parallel_2',46.0],PARAMETER['Latitude_Of_Origin',43.66666666666666],UNIT['Foot',0.3048]]", transform_method=["NAD_1983_HARN_To_WGS_1984_2"], in_coor_system="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]"""
print("Reprojecting to \"%s\"." % output_fc)
#https://pro.arcgis.com/en/pro-app/latest/tool-reference/data-management/project.htm
arcpy.management.Project(
wm_fc,
output_fc,
out_coor_system=sref,
#transform_method='NAD_1983_HARN_To_WGS_1984_2'
)
return output_fc
def process_gnis_points(input_fc, output_fc):
""" Take a GNIS feature class as input.
Clean it.
Write a feature class in local projection.
Returns the name of the feature class.
Uses the workspace as the output location. """
# State GNIS data arrives in OGIC projection
# Filter on Clatsop County
or_df = GeoAccessor.from_featureclass(input_fc)
sdf = or_df[or_df.COUNTY_NAME == "Clatsop"]
#https://pro.arcgis.com/en/pro-app/latest/arcpy/classes/spatialreference.htm
#sref = arcpy.SpatialReference()
#rval = sref.loadFromString('EPSG:2913')
pd.set_option('display.max_columns', None)
print(sdf.head(5))
print("Rows:", len(sdf))
print("Columns: ", sdf.columns)
print("Types:", sdf.dtypes)
# Columns that we're keeping
# There are a bunch ignored including COUNTY STATE COUNTRY
# and specialize ones that probably only matter to GeoComm like rSrcId
# There are two GlobalId fields, huh. That's annoying. I keep only 1.
sdf = sdf[[
'SHAPE',
'FEATURE_ID', # a long int
'FEATURE_NAME', #
"FEATURE_CLASS", # Make the domain from this
'ELEV_IN_FT',
# 'MAP_NAME', # Quad?? "Warrenton", "Tillamook Head"...
"DATE_CREATED",
"DATE_EDITED",
"Elevation_Source", # "GNIS", "LIDAR DEM", "10m DEM", ...
]]
# Rename some fields
sdf.rename(columns={
'FEATURE_NAME': 'name',
'FEATURE_CLASS': 'category',
'DATE_CREATED': 'created_date',
'DATE_EDITED': 'last_edited_date',
},
inplace=True)
# Add some fields and set defaults on them.
sdf.loc[:, 'created_user'] = '******'
sdf.loc[:, 'last_edited_user'] = ''
sdf.loc[:, 'source'] = 'gnis'
sdf.loc[:, 'subcategory'] = sdf.loc[:, 'category']
# Change float64 column to string, to remove the stupid ".0" endings.
sdf[['FEATURE_ID']] = sdf[['FEATURE_ID']].astype("int64")
sdf[['created_date']] = sdf[['created_date']].astype("datetime64")
sdf[['last_edited_date']] = sdf[['last_edited_date']].astype("datetime64")
print("We have %d points." % len(sdf))
print(sdf.head(5))
print(sdf.columns)
print(sdf.dtypes)
fc = output_fc + '_ogic'
sdf.spatial.to_featureclass(fc, sanitize_columns=False)
print("Wrote %d points to \"%s\"." % (len(sdf), fc))
# Reproject web mercator points to local.
# THIS METHOD DOES NOT WORK -- it spins forever
# if you remove the transform it works but generates bad data
# local_df = sdf.copy()
# rval = local_df.spatial.project(2913, 'NAD_1983_HARN_To_WGS_1984_2')
# print("LOCAL", sdf.spatial.sr, local_df.spatial.sr, rval)
# write_fc(local_df, local_fc)
# I got this from a Model export
# Using a transform string just seems to cause the process to lock up
# and this sref string apparently includes the transform.
sref = """PROJCS['NAD_1983_HARN_StatePlane_Oregon_North_FIPS_3601_Feet_Intl',GEOGCS['GCS_North_American_1983_HARN',DATUM['D_North_American_1983_HARN',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Lambert_Conformal_Conic'],PARAMETER['False_Easting',8202099.737532808],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',-120.5],PARAMETER['Standard_Parallel_1',44.33333333333334],PARAMETER['Standard_Parallel_2',46.0],PARAMETER['Latitude_Of_Origin',43.66666666666666],UNIT['Foot',0.3048]]", transform_method=["NAD_1983_HARN_To_WGS_1984_2"], in_coor_system="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]"""
print("Reprojecting to \"%s\"." % output_fc)
#https://pro.arcgis.com/en/pro-app/latest/tool-reference/data-management/project.htm
arcpy.management.Project(
fc,
output_fc,
out_coor_system=sref,
#transform_method='NAD_1983_HARN_To_WGS_1984_2'
)
return output_fc
poi_name = "points_of_interest"
# Input data
cc_points = egdb + "/" + 'Clatsop.DBO.' + poi_name + '_OLD'
assert arcpy.Exists(cc_points)
gnis_points = "K:/State Data/GNIS_2014/GNIS_OR.gdb/GNIS_OR.gdb/GNIS_OR_Buffer"
assert arcpy.Exists(gnis_points)
pd.set_option('display.max_columns', None)
# Insert the name of the existing POI fc including datestamp to speed development
current_places = fgdb + '/' + 'poi_name_20210505_1507' # put datestamp here from previous run
if arcpy.Exists(current_places):
sdf = GeoAccessor.from_featureclass(current_places)
else:
poi_fc = fgdb + '/cc_points'
process_cc_poi(cc_points, poi_fc)
gnis_fc = fgdb + '/gnis_points'
process_gnis_points(gnis_points, gnis_fc)
# Cat them together
df1 = GeoAccessor.from_featureclass(poi_fc)
df2 = GeoAccessor.from_featureclass(gnis_fc)
sdf = pd.concat([df1, df2])
places = fgdb + '/' + poi_name + suffix
# Clean!
# Add city as an attribute to POIs, to make for better Locator results
def _get_closest_df_arcpy(origin_df,
dest_df,
dest_count,
network_dataset,
max_dist=None):
"""
Succinct function wrapping find_closest_facilities with a little ability to handle network and server hiccups
:param origin_df: Origin points Spatially Enabled Dataframe
:param dest_df: Destination points Spatially Enabled Dataframe
:param dest_count: Destination points Spatially Enabled Dataframe
:param network_dataset: Path to ArcGIS Network dataset.
:param max_dist: Maximum nearest routing distance in miles.
:return: Spatially Enabled Dataframe of solved closest facility routes.
"""
# get the mode of travel from the network dataset - rural so gravel roads are fair game
nd_lyr = arcpy.nax.MakeNetworkDatasetLayer(network_dataset)[0]
trvl_mode_dict = arcpy.nax.GetTravelModes(nd_lyr)
trvl_mode = trvl_mode_dict['Rural Driving Time']
# create the closest solver object instance
# https://pro.arcgis.com/en/pro-app/arcpy/network-analyst/closestfacility.htm
closest_solver = arcpy.nax.ClosestFacility(network_dataset)
# set parameters for the closest solver
closest_solver.travelMode = trvl_mode
closest_solver.travelDirection = arcpy.nax.TravelDirection.ToFacility
# TODO: How to set this to distance?
closest_solver.timeUnits = arcpy.nax.TimeUnits.Minutes
closest_solver.distanceUnits = arcpy.nax.DistanceUnits.Miles
closest_solver.defaultTargetFacilityCount = dest_count
closest_solver.routeShapeType = arcpy.nax.RouteShapeType.TrueShapeWithMeasures
closest_solver.searchTolerance = 5000
closest_solver.searchToleranceUnits = arcpy.nax.DistanceUnits.Meters
# since maximum distance is optional, well, make it optional
if max_dist:
closest_solver.defaultImpedanceCutoff = max_dist
# load the origin and destination feature data frames into memory and load into the solver object instance
origin_fc = origin_df.spatial.to_featureclass(
os.path.join(arcpy.env.scratchGDB, 'origin'))
closest_solver.load(arcpy.nax.ClosestFacilityInputDataType.Incidents,
origin_fc)
dest_fc = dest_df.spatial.to_featureclass(
os.path.join(arcpy.env.scratchGDB, 'dest'))
closest_solver.load(arcpy.nax.ClosestFacilityInputDataType.Facilities,
dest_fc)
# run the solve, and get comfortable
closest_result = closest_solver.solve()
# export the results to a spatially enabled data frame
route_fc = 'in_memory/routes'
closest_result.export(arcpy.nax.ClosestFacilityOutputDataType.Routes,
route_fc)
route_oid_col = arcpy.Describe(route_fc).OIDFieldName
closest_df = GeoAccessor.from_featureclass(route_fc)
if route_oid_col:
closest_df.drop(columns=[route_oid_col], inplace=True)
arcpy.management.Delete(route_fc)
# get rid of the extra empty columns the local network solve adds
closest_df.dropna(axis=1, how='all', inplace=True)
# populate the origin and destination fields so the schema matches what online solve returns
name_srs = closest_df.Name.str.split(' - ')
closest_df['IncidentID'] = name_srs.apply(lambda val: val[0])
closest_df['FacilityID'] = name_srs.apply(lambda val: val[1])
return closest_df
def _get_nearest_solve_local(origin_dataframe: pd.DataFrame,
destination_dataframe: pd.DataFrame,
destination_count: int,
network_dataset: [Path, str],
maximum_distance: [int, float] = None):
"""
Perform network solve using local resources with assumption of standard input.
Args:
origin_dataframe: Origin points Spatially Enabled Dataframe
destination_dataframe: Destination points Spatially Enabled Dataframe
destination_count: Destination points Spatially Enabled Dataframe
network_dataset: Path to ArcGIS Network dataset for performing routing.
maximum_distance: Maximum nearest routing distance in miles.
Returns: Spatially Enabled Dataframe of solved closest facility routes.
"""
# make sure the path to the network dataset is a string
network_dataset = str(network_dataset) if isinstance(
network_dataset, Path) else network_dataset
# get the mode of travel from the network dataset - rural so gravel roads are fair game
nd_lyr = arcpy.nax.MakeNetworkDatasetLayer(network_dataset)[0]
trvl_mode_dict = arcpy.nax.GetTravelModes(nd_lyr)
trvl_mode = trvl_mode_dict['Rural Driving Time']
# create the closest solver object instance
# https://pro.arcgis.com/en/pro-app/arcpy/network-analyst/closestfacility.htm
closest_solver = arcpy.nax.ClosestFacility(network_dataset)
# set parameters for the closest solver
closest_solver.travelMode = trvl_mode
closest_solver.travelDirection = arcpy.nax.TravelDirection.ToFacility
# TODO: How to set this to distance?
closest_solver.timeUnits = arcpy.nax.TimeUnits.Minutes
closest_solver.distanceUnits = arcpy.nax.DistanceUnits.Miles
closest_solver.defaultTargetFacilityCount = destination_count
closest_solver.routeShapeType = arcpy.nax.RouteShapeType.TrueShapeWithMeasures
closest_solver.searchTolerance = 5000
closest_solver.searchToleranceUnits = arcpy.nax.DistanceUnits.Meters
# since maximum distance is optional, well, make it optional
if maximum_distance is not None:
closest_solver.defaultImpedanceCutoff = maximum_distance
# load the origin and destination feature data frames into memory and load into the solver object instance
# TODO: test if can use 'memory' workspace instead of scratch
origin_fc = origin_dataframe.spatial.to_featureclass(
os.path.join(arcpy.env.scratchGDB, 'origin_tmp'))
closest_solver.load(arcpy.nax.ClosestFacilityInputDataType.Incidents,
origin_fc)
dest_fc = destination_dataframe.spatial.to_featureclass(
os.path.join(arcpy.env.scratchGDB, 'dest_tmp'))
closest_solver.load(arcpy.nax.ClosestFacilityInputDataType.Facilities,
dest_fc)
# run the solve, and get comfortable
closest_result = closest_solver.solve()
# export the results to a spatially enabled data frame, and do a little cleanup
# TODO: test if can use 'memory/routes' instead - the more current method
route_fc = 'in_memory/routes'
closest_result.export(arcpy.nax.ClosestFacilityOutputDataType.Routes,
route_fc)
route_oid_col = arcpy.Describe(route_fc).OIDFieldName
closest_df = GeoAccessor.from_featureclass(route_fc)
arcpy.management.Delete(route_fc)
if route_oid_col:
closest_df.drop(columns=[route_oid_col], inplace=True)
# get rid of the extra empty columns the local network solve adds
closest_df.dropna(axis=1, how='all', inplace=True)
# populate the origin and destination fields so the schema matches what online solve returns
name_srs = closest_df.Name.str.split(' - ')
closest_df['IncidentID'] = name_srs.apply(lambda val: val[0])
closest_df['FacilityID'] = name_srs.apply(lambda val: val[1])
return closest_df
from arcgis.features import GeoAccessor
from arcgis.gis import GIS
import pandas as pd
import time
from osm_runner import Runner
if __name__ == "__main__":
# Load Extent from Local Shapefile
sdf = GeoAccessor.from_featureclass('..\\data\\DC\\DC.shp')
ext = sdf.iloc[0]['SHAPE'].extent
# Format Extent for Overpass API (S, W, N, E)
bbox = f'({ext[1]},{ext[0]},{ext[3]},{ext[2]})'
# Get Instance of OSM Runner
runner = Runner()
# Fetch Surveillance Cameras
df = runner.gen_osm_df('point', bbox, 'man_made')
# Connect to GIS
gis = GIS('***', '***', '***')
# Dump HFL to Enterprise
df.spatial.to_featurelayer(f'OSM Cameras {round(time.time())}', gis=gis)
def to_sdf(self) -> pd.DataFrame:
# convert the layer to a spatially enabled dataframe
df = GeoAccessor.from_featureclass(self)
# get rid of the object id field and return the dataframe
return df.drop('OBJECTID', axis=1)
def df_bp():
df_bp = GeoAccessor.from_featureclass(str(pth_bp))
df_bp = df_bp[['FIPS', 'SHAPE', 'HH_C']].copy()
df_bp.columns = ['FIPS', 'SHAPE', 'HouseholdCount']
return df_bp
def get_dataframe(in_features, gis=None):
"""
Get a spatially enabled dataframe from the input features provided.
:param in_features: Spatially Enabled Dataframe | String path to Feature Class | pathlib.Path object to feature
class | ArcGIS Layer object |String url to Feature Service | String Web GIS Item ID
Resource to be evaluated and converted to a Spatially Enabled Dataframe.
:param gis: Optional GIS object instance for connecting to resources.
"""
# if a path object, convert to a string for following steps to work correctly
in_features = str(in_features) if isinstance(in_features, pathlib.Path) else in_features
# helper for determining if feature layer
def _is_feature_layer(in_ftrs):
if hasattr(in_ftrs, 'isFeatureLayer'):
return in_ftrs.isFeatureLayer
else:
return False
# if already a Spatially Enabled Dataframe, mostly just pass it straight through
if isinstance(in_features, pd.DataFrame) and in_features.spatial.validate() is True:
df = in_features
# if a csv previously exported from a Spatially Enabled Dataframe, get it in
elif isinstance(in_features, str) and os.path.exists(in_features) and in_features.endswith('.csv'):
df = pd.read_csv(in_features)
df['SHAPE'] = df['SHAPE'].apply(lambda geom: Geometry(eval(geom)))
# this almost always is the index written to the csv, so taking care of this
if df.columns[0] == 'Unnamed: 0':
df = df.set_index('Unnamed: 0')
del (df.index.name)
# create a Spatially Enabled Dataframe from the direct url to the Feature Service
elif isinstance(in_features, str) and in_features.startswith('http'):
# submitted urls can be lacking a few essential pieces, so handle some contingencies with some regex matching
regex = re.compile(r'((^https?://.*?)(/\d{1,3})?)\?')
srch = regex.search(in_features)
# if the layer index is included, still clean by dropping any possible trailing url parameters
if srch.group(3):
in_features = f'{srch.group(1)}'
# ensure at least the first layer is being referenced if the index was forgotten
else:
in_features = f'{srch.group(2)}/0'
# if the layer is unsecured, a gis is not needed, but we have to handle differently
if gis is not None:
df = FeatureLayer(in_features, gis).query(out_sr=4326, as_df=True)
else:
df = FeatureLayer(in_features).query(out_sr=4326, as_df=True)
# create a Spatially Enabled Dataframe from a Web GIS Item ID
elif isinstance(in_features, str) and len(in_features) == 32:
# if publicly shared on ArcGIS Online this anonymous gis can be used to access the resource
if gis is None:
gis = GIS()
itm = gis.content.get(in_features)
df = itm.layers[0].query(out_sr=4326, as_df=True)
elif isinstance(in_features, (str, pathlib.Path)):
df = GeoAccessor.from_featureclass(in_features)
# create a Spatially Enabled Dataframe from a Layer
elif _is_feature_layer(in_features):
df = FeatureSet.from_json(arcpy.FeatureSet(in_features).JSON).sdf
# sometimes there is an issue with modified or sliced dataframes with the SHAPE column not being correctly
# recognized as a geometry column, so try to set it as the geometry...just in case
elif isinstance(in_features, pd.DataFrame) and 'SHAPE' in in_features.columns:
in_features.spatial.set_geometry('SHAPE')
df = in_features
if df.spatial.validate() is False:
raise Exception('Could not process input features for get_dataframe function. Although the input_features '
'appear to be in a Pandas Dataframe, the SHAPE column appears to not contain valid '
'geometries. The Dataframe is not validating using the *.spatial.validate function.')
else:
raise Exception('Could not process input features for get_dataframe function.')
# ensure the universal spatial column is correctly being recognized
df.spatial.set_geometry('SHAPE')
return df
def process_address_points(input_fc, output_fc):
# NOTE, Geocomm data arrives in Web Mercator.
sdf = GeoAccessor.from_featureclass(input_fc)
#https://pro.arcgis.com/en/pro-app/latest/arcpy/classes/spatialreference.htm
#sref = arcpy.SpatialReference()
#rval = sref.loadFromString('EPSG:2913')
print(len(sdf))
print(sdf.columns)
print(sdf.dtypes)
# Clean!
# Change float64 columns to strings, removing the stupid ".0" endings.
# You have to fix the values first as there are some entries with spaces in them.
sdf = sdf.apply(fixint, axis=1)
# Converting to int64 fails here because some of the rows have empty strings.
sdf.loc[:, 'addNum'].astype("str")
sdf.loc[:, 'zipCode'].astype("str")
# Columns that we're keeping
# There are a bunch ignored including COUNTY STATE COUNTRY
# and specialize ones that probably only matter to GeoComm like rSrcId
# There are two GlobalId fields, huh. That's annoying. I keep only 1.
sdf = sdf[[
'SHAPE',
'srcFullAdr', # "1234 1/2 SOUTHWEST VISTA DEL MAR DRIVE"
'gcLgFlAdr', # "197 N MARION AVE" -- shortened with abbreviations
'gcLgFlName', # "N MARION AVE"
'gcFullName', # "NORTH MARION AVENUE"
#'addNumPre', # At least this year, this field is empty.
'addNum',
'addNumSuf', # "1020 1/2" or "1020 A" -- addNum is always an integer
'preMod', # "ALTERNATE" or "OLD" or "VISTA"
'preDir', # spelled out like SOUTHWEST
'preType', # 'HIGHWAY'
'preTypeSep', # "DEL"
'strName', # "30" as in "ALTERNATE HIGHWAY 30"
'postType', # "DRIVE"
'postDir', # "SOUTH"
'postMod', # "BUSINESS"
# 'building', 'floor', 'room', 'seat', # These are EMPTY so far
# 'location', 'landmark', 'placeType', # ALL EMPTY
'unit',
'unitDesc',
'unitNo', # "UNIT 208|APT 5", "UNIT|SUITE|APARTMENT", "208"
'incMuni', # City name or "UNINCORPORATED"
'msagComm', # City or locality such as 'ASTORIA' or 'BIRKENFELD' or 'WESTPORT'
'postComm', # The post office, for example BIRKENFELD and WESTPORT go to CLATSKANIE
#'unincComm', 'nbrhdComm', # ALL EMPTY
'zipCode',
# 'zipCode4', # ALL EMPTY
# 'esn', # THIS IS INFORMATION FOR DISPATCH I DID NOT INCLUDE
# 'elev', # EMPTY
# 'milepost', # is EMPTY
'long',
'lat',
'srcLastEdt',
'EditDate', # I think this corresponds to GeoComm update date
'gcConfiden', # Confidence, 1 = good
'GlobalID_2',
'taxlotID' # GlobalId is null for a few entries so I use "_2"
]]
# Create aliases
# The original column names are left untouched,
# but I assign more readable aliases here.
# NB some of these names are chosen to match up with the Esri point_address locator.
# NB It turns out assigning some them in the locator makes the locator results icky.
d_aliases = {
'srcFullAdr': 'Full Address Long',
'gcLgFlAdr': 'Full Address',
'gcLgFlName': 'Street Name',
'gcFullName': 'Full Street Name',
'addNum': 'House Number',
'addNumSuf': 'Address Number Suffix',
'preMod': 'Prefix Modifier',
'preDir': 'Prefix Direction',
'preType': 'Prefix type',
'preTypeSep': 'Prefix Type Separator',
'strName': 'Street',
'postType': 'Postfix Type',
'postDir': 'Postfix Direction',
'postMod': 'Postfix Modifier',
'unit': 'Unit',
'unitDesc': 'Unit Description',
'unitNo': 'Unit Number',
'incMuni': 'City',
'msagComm': 'MSAG Community',
'postComm': 'Postal Community',
'zipCode': 'ZIP',
'long': 'Longitude',
'lat': 'Latitude',
'srcLastEdt': 'Edit Date',
'EditDate': 'Upload Date',
'gcConfiden': 'Confidence',
'GlobalID_2': 'GUID',
'taxlotID': 'Taxlot ID'
}
print("We have %d address points." % len(sdf))
print(sdf.head(5))
print(sdf.columns)
print(sdf.dtypes)
# Read the parcel db; all I want is a foreign key.
# BUT, there is already one in there so skip this
# parcel_sdf = GeoAccessor.from_featureclass(parcel_fc)[['ACCOUNT_ID', 'SHAPE']]
# parcel_sdf.rename(columns={'ACCOUNT_ID' : 'parcel_join_id'}, inplace=True)
# sdf = sdf.spatial.join(parcel_sdf, how='left', op='intersects')
# print(sdf.dtypes)
# print(sdf.head(4))
output_wm = output_fc + '_wm'
sdf.spatial.to_featureclass(output_wm, sanitize_columns=False)
set_aliases(output_wm, d_aliases)
print("Wrote %d points to \"%s\"." % (len(sdf), output_wm))
# Reproject web mercator points to local.
# THIS METHOD DOES NOT WORK -- it spins forever
# if you remove the transform it works but generates bad data
# local_df = sdf.copy()
# rval = local_df.spatial.project(2913, 'NAD_1983_HARN_To_WGS_1984_2')
# print("LOCAL", sdf.spatial.sr, local_df.spatial.sr, rval)
# write_fc(local_df, local_fc)
# I got this from a Model export
# Using a transform string just seems to cause the process to lock up
# and this sref string apparently includes the transform.
sref = """PROJCS['NAD_1983_HARN_StatePlane_Oregon_North_FIPS_3601_Feet_Intl',GEOGCS['GCS_North_American_1983_HARN',DATUM['D_North_American_1983_HARN',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Lambert_Conformal_Conic'],PARAMETER['False_Easting',8202099.737532808],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',-120.5],PARAMETER['Standard_Parallel_1',44.33333333333334],PARAMETER['Standard_Parallel_2',46.0],PARAMETER['Latitude_Of_Origin',43.66666666666666],UNIT['Foot',0.3048]]", transform_method=["NAD_1983_HARN_To_WGS_1984_2"], in_coor_system="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]"""
print("Reprojecting to \"%s\"." % output_fc)
#https://pro.arcgis.com/en/pro-app/latest/tool-reference/data-management/project.htm
arcpy.management.Project(
output_wm,
output_fc,
out_coor_system=sref,
#transform_method='NAD_1983_HARN_To_WGS_1984_2'
)
set_aliases(output_fc, d_aliases)
def process_hydrants(input_fc, output_fc):
# NOTE, Geocomm data arrives in Web Mercator.
sdf = GeoAccessor.from_featureclass(input_fc)
#https://pro.arcgis.com/en/pro-app/latest/arcpy/classes/spatialreference.htm
#sref = arcpy.SpatialReference()
#rval = sref.loadFromString('EPSG:2913')
print(len(sdf))
print(sdf.columns)
print(sdf.dtypes)
# Clean!
# Change float64 columns to strings, removing the stupid ".0" endings.
sdf = sdf
#sdf = sdf.apply(fixid, axis=1)
# Columns that we're keeping
# There are a bunch ignored including COUNTY STATE COUNTRY
sdf = sdf[[
'SHAPE',
'HYDRANTID',
'STREET_INT', # "NW CORNER"
'LOC_DESC', # 'BIG CREEK HATCHERY'
'ADDRESS', # '93011 LABECK RD'
'AUTHORITY', # "CITY OF ASTORIA PUBLIC WORKS"
'COMMUNITY', # 'ASTORIA'
'Source', # 'BURNSIDE WATER ASSN HYDRANTS.DOC'
]]
# Create aliases
# The original column names are left untouched,
# but I assign more readable aliases here.
d_aliases = {
'HYDRANTID': 'Hydrant ID',
'STREET_INT': 'Intersection',
'LOC_DESC': 'Description',
'ADDRESS': 'Address',
'AUTHORITY': 'Authority',
'COMMUNITY': 'Community',
}
print("We have %d points." % len(sdf))
print(sdf.head(5))
print(sdf.columns)
print(sdf.dtypes)
wm_fc = output_fc + '_wm'
sdf.spatial.to_featureclass(wm_fc, sanitize_columns=False)
set_aliases(wm_fc, d_aliases)
print("Wrote %d points to \"%s\"." % (len(sdf), wm_fc))
# Reproject web mercator points to local.
# THIS METHOD DOES NOT WORK -- it spins forever
# if you remove the transform it works but generates bad data
# local_df = sdf.copy()
# rval = local_df.spatial.project(2913, 'NAD_1983_HARN_To_WGS_1984_2')
# print("LOCAL", sdf.spatial.sr, local_df.spatial.sr, rval)
# write_fc(local_df, local_fc)
# I got this from a Model export
# Using a transform string just seems to cause the process to lock up
# and this sref string apparently includes the transform.
sref = """PROJCS['NAD_1983_HARN_StatePlane_Oregon_North_FIPS_3601_Feet_Intl',GEOGCS['GCS_North_American_1983_HARN',DATUM['D_North_American_1983_HARN',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Lambert_Conformal_Conic'],PARAMETER['False_Easting',8202099.737532808],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',-120.5],PARAMETER['Standard_Parallel_1',44.33333333333334],PARAMETER['Standard_Parallel_2',46.0],PARAMETER['Latitude_Of_Origin',43.66666666666666],UNIT['Foot',0.3048]]", transform_method=["NAD_1983_HARN_To_WGS_1984_2"], in_coor_system="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]"""
print("Reprojecting to \"%s\"." % output_fc)
#https://pro.arcgis.com/en/pro-app/latest/tool-reference/data-management/project.htm
arcpy.management.Project(
wm_fc,
output_fc,
out_coor_system=sref,
#transform_method='NAD_1983_HARN_To_WGS_1984_2'
)
set_aliases(output_fc, d_aliases)
# variables
csv = r"C:\jescudero\Esri\AM\data\erradicacion_05_08.csv"
erradicacion_id = "9bc352654401432ca046825fd35f8fc1"
# Read data
names = [
"nombredepartamento", "nombremunicipio", 'hectareas', 'grupoarmado',
"latitud", "longitud"
]
df = pd.read_csv(csv,
sep=",",
usecols=[13, 14, 21, 31, 43, 44],
names=names,
skiprows=1)
# Connect to gis
gis = GIS(agol_url, agol_user, agol_password)
# Prepare data
df_xy = GeoAccessor.from_xy(df, "longitud", "latitud", sr=4326)
fs = df_xy.spatial.to_featureset()
# Append data
erradicacion_item = gis.content.get(erradicacion_id)
result = erradicacion_item.layers[0].edit_features(adds=fs)
print(result)
# End
def get_country_api_data(
protocol,
start_date=start_date,
end_date=end_date,
is_clean=True,
countries=[],
regions=[],
):
"""
Gets country enriched API Data. Due note that this data comes from layers in ArcGIS that are updated daily. Therefore, there will be some delay between when an entry is uploaded onto the GLOBE data base and being on the ArcGIS dataset.
Parameters
----------
protocol : str, {"mosquito_habitat_mapper", "land_covers"}
The desired GLOBE Observer Protocol. Currently only mosquito habitat mapper and land cover is supported.
start_date : str, default= 2017-05-31
The desired start date of the dataset in the format of (YYYY-MM-DD).
end_date : str, default= today's date in YYYY-MM-DD form.
The desired end date of the dataset in the format of (YYYY-MM-DD).
countries : list of str, default=[]
The list of desired countries. Look at go_utils.info.region_dict to see supported country names. If the list is empty, all data will be included.
regions : list of str, default=[]
The list of desired regions. Look at go_utils.info.region_dict to see supported region names and the countries they enclose. If the list is empty, all data will be included.
latlon_box : dict of {str, double}, optional
The longitudes and latitudes of a bounding box for the dataset. The minimum/maximum latitudes and longitudes must be specified with the following keys: "min_lat", "min_lon", "max_lat", "max_lon". The default value specifies all latitude and longitude coordinates.
"""
item_id_dict = {
mosquito_protocol: "a018521fbc3f42bc848d3fa4c52e02ce",
landcover_protocol: "fe54b831415f44d2b1640327ae276fb8",
}
if protocol not in item_id_dict:
raise ValueError(
"Invalid protocol, currently only 'mosquito_habitat_mapper' and 'land_covers' are supported."
)
gis = GIS()
item = gis.content.get(itemid=item_id_dict[protocol])
df = GeoAccessor.from_layer(item.layers[0])
if "SHAPE" in df:
df.drop(["SHAPE"], axis=1, inplace=True)
# Due to the size of the mhm column names, ArcGIS truncates the names so it must be renamed in this step.
if protocol == "mosquito_habitat_mapper":
mhm_rename_dict = {
col: f"mosquitohabitatmapper{col}"
for col in df.columns if col[0].isupper() and col != "COUNTRY"
}
df.rename(
mhm_rename_dict,
axis=1,
inplace=True,
)
# Filter the dates
start = datetime.strptime(start_date, "%Y-%m-%d")
end = datetime.strptime(end_date, "%Y-%m-%d")
measured_at = protocol.replace("_", "") + "MeasuredAt"
convert_dates_to_datetime(df)
df = df[(df[measured_at] >= start) & (df[measured_at] <= end)]
if is_clean:
df = default_data_clean(df, protocol)
for region in regions:
countries.extend(region_dict[region])
countries_set = set(countries)
# Return the regular data if nothing is specified
if not countries_set:
return df
else:
mask = _get_valid_countries_mask(df, protocol, countries_set)
return df[mask]
def _enrich_wrapper(enrich_var_lst: list,
feature_class_to_enrich: str,
logger: logging.Logger,
id_field: str = None,
input_feature_class_fields_in_output: bool = False,
return_geometry: bool = False,
enrich_threshold: int = 1000) -> pd.DataFrame:
"""Wrapper around Enrich function to make it work"""
# ensure using local data
data.set_to_usa_local()
# if an ID field is provided, use it, but if not, use the OID field
id_fld = id_field if id_field else arcpy.Describe(
feature_class_to_enrich).OIDFieldName
# ensure the path is being used for the input feature class since it could be a layer
enrich_fc = arcpy.Describe(feature_class_to_enrich).catalogPath
out_gdb = arcpy.env.scratchGDB
# since the Enrich tool pukes with too much data, get the count and batch the process if necessary
feature_count = int(arcpy.management.GetCount(enrich_fc)[0])
if feature_count > enrich_threshold:
# create a list of individual query statements for each feature
oid_list = [r[0] for r in arcpy.da.SearchCursor(enrich_fc, 'OID@')]
oid_fld = arcpy.Describe(enrich_fc).OIDFieldName
query_lst = [f'{oid_fld} = {oid}' for oid in oid_list]
# break the list into chunks to select features in chunks up to the enrich threshold
oid_chunk_lst = blow_chunks(query_lst, enrich_threshold)
query_chunk_lst = [' OR '.join(chunk) for chunk in oid_chunk_lst]
# list to put all the chunk processed enrichment feature classes
enriched_chunk_lst = []
logger.debug(
f'Splitting the enrich task into {len(query_chunk_lst)} chunks.')
# iterate the query chunks and create the chunked analysis areas
for enrich_idx, query_chunk in enumerate(query_chunk_lst):
logger.debug(
f'Starting to enrich {enrich_idx+1}/{len(query_chunk_lst)}')
# extract out a temporary dataset into memory for this loop
tmp_features = arcpy.analysis.Select(
in_features=feature_class_to_enrich,
out_feature_class=os.path.join(
out_gdb, f'in_enrich_chunk_{enrich_idx:04d}'),
where_clause=query_chunk)[0]
# write the run result to the output geodatabase
enrich_fc = arcpy.ba.EnrichLayer(
in_features=tmp_features,
out_feature_class=os.path.join(out_gdb,
f'enrich_fc_{enrich_idx:04d}'),
variables=';'.join(enrich_var_lst))[0]
# delete the temporary input data
arcpy.management.Delete(tmp_features)
# add this iteration to the enriched chunks
enriched_chunk_lst.append(enrich_fc)
logger.debug(
f'Finished enriching {enrich_idx + 1}/{len(query_chunk_lst)}')
# combine all the chunked outputs together
enrich_fc = arcpy.management.Merge(
enriched_chunk_lst, os.path.join(out_gdb, 'enriched_data'))
# take out the trash
for fc in enriched_chunk_lst:
arcpy.management.Delete(fc)
else:
enrich_fc = arcpy.ba.EnrichLayer(in_features=enrich_fc,
out_feature_class=os.path.join(
out_gdb, 'enriched_data'),
variables=';'.join(enrich_var_lst))[0]
# convert the output to a Spatially Enabled Dataframe
enrich_df = GeoAccessor.from_featureclass(enrich_fc)
# remove the temporary data
arcpy.management.Delete(enrich_fc)
# start putting together a list of fields to drop for cleanup
drop_fld_lst = ['aggregationMethod']
# if only desiring the enrich variables in the output
if not input_feature_class_fields_in_output:
drop_fld_lst.extend([
f.name for f in arcpy.ListFields(feature_class_to_enrich)
if f.name != id_fld and f.name.upper() != 'SHAPE'
])
# if geometry is not desired as an output
if not return_geometry:
drop_fld_lst.append('SHAPE')
# ensure the columns to drop are actually in the dataframe
drop_fld_lst = [
col for col in drop_fld_lst if col in list(enrich_df.columns)
]
# clean up the dataframe fields
enrich_df.drop(drop_fld_lst, axis=1, inplace=True)
return enrich_df