def transform(self):
"""Transforms addresses.csv
Transforms addresses.csv to new_addresses.csv by adding geocoded x, y coordinates to new_addresses.csv.
Returns:
./data/new_addresses.csv with geocoded x, y coordinates for use with arcpy.
"""
arcpy.AddMessage('Transforming addresses using geocoder')
# Create transformed csv with X, Y, and Type for headers
csv_path = set_path(self.config_dict['etl_dir'], 'addresses.csv')
new_csv_path = set_path(self.config_dict['etl_dir'], 'new_addresses.csv')
with open(new_csv_path, 'w', newline='', encoding='utf-8') as f:
fieldnames = ['X', 'Y', 'Type']
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
with open(csv_path, 'r') as address_reader:
csv_dict = csv.DictReader(address_reader, delimiter=',')
for row in csv_dict:
address = row['Address']
arcpy.AddMessage(f'geocoding {address}')
url = f"{self.config_dict['geocoder_prefix_url']}{address}{self.config_dict['geocoder_suffix_url']}"
r = self.s.get(url)
resp_dict = r.json()
x = resp_dict['result']['addressMatches'][0]['coordinates']['x']
y = resp_dict['result']['addressMatches'][0]['coordinates']['y']
row_dict = {'X': x, 'Y': y, 'Type': 'Residential'}
arcpy.AddMessage(f'Writing row to new_addresses.csv: {row_dict}')
writer.writerow(row_dict)
def render_layout(map_subtitle, map_features, map_spatial_reference,
address_count, output_db):
"""Map renderer orchestration.
Orchestrates the rendering of layout and converting the layout to a pdf output file.
Args:
map_subtitle: Desired subtitle for the output map.
map_features: Desired features to map.
map_spatial_reference: Desired spatial reference for map.
address_count: Addresses count variable previously calculated for output map.
output_db: File location for map_features.
Returns:
Side effect is rendering functions are provided with correct inputs for layout configuration and pdf file output.
"""
aprx_path = set_path(config_dict.get('proj_dir'), 'WestNileOutbreak.aprx')
aprx = arcpy.mp.ArcGISProject(aprx_path)
arcpy.AddMessage(f'aprx path: {aprx.filePath}')
mp = get_map(aprx, 'Map')
set_spatial_reference(mp, map_spatial_reference)
for f, c in map_features:
fc_name = f
fc = set_path(output_db, f)
colour = c
add_feature_to_map(mp, fc_name, fc, colour, transparency=50)
# Export final map
export_map(aprx, map_subtitle, address_count)
def export_map(subtitle):
logger.debug('Starting map export.')
# Setup aprx
aprx_path = set_path(config_dict.get('proj_dir'), 'WestNileOutbreak.aprx')
aprx = arcpy.mp.ArcGISProject(aprx_path)
lyt = aprx.listLayouts()[0]
for el in lyt.listElements():
arcpy.AddMessage(el.name)
if 'Title' in el.name:
el.text = f'{el.text} {subtitle}'
arcpy.AddMessage(el.text)
aprx.save()
lyt.exportToPDF(f'{config_dict["proj_dir"]}/wnv.pdf')
logger.debug('Export map complete.')
def main(flush_output_db=False):
"""main orchestration
Coordinates the major operations of the West Nile Outbreak project.
No try except block included because these exist in helper functions called by this function.
Args:
flush_output_db:
=False means contents of the output db will not be deleted.
=True means contents of the output db will be deleted
"""
# Setup geoprocessing environment.
# NAD 1983 StatePlane Colorado North: https://www.spatialreference.org/ref/esri/102653/
pcs = 102653
# Setup arcgis environment
arcgis_setup(flush_output_db, spatial_reference=pcs)
# Setup output db
output_db = config_dict.get('output_gdb_dir')
arcpy.AddMessage(f'output db: {output_db}')
# ----- run_etl -----
# Run etl, generates the avoid_points feature class.
run_etl()
# ----- run_analysis -----
# Run Analysis to create the final analysis features.
analysis_results_dictionary = run_analysis(output_db)
# ----- render_layout -----
# Render the map including analysis features, correct colours, subtitle, and addresses at risk count.
# analysis_results_dictionary below is included for debugging so don't have to run_analysis:
# analysis_results_dictionary = {'map_subtitle': 'debug subtitle', 'addresses_at_risk_count': 123}
map_features = [('final_analysis', [255, 0, 0, 100]),
('avoid_points_buf', [115, 178, 255, 100]),
('Target_Addresses', [102, 119, 205, 100])]
map_subtitle = analysis_results_dictionary['map_subtitle']
map_spatial_reference = pcs
address_count = analysis_results_dictionary['addresses_at_risk_count']
render_layout(map_subtitle, map_features, map_spatial_reference,
address_count, output_db)
# ----- generate_report -----
# Generate a csv report in the WestNileOutbreak directory with the Target Addresses that require spraying.
target_addresses_fc = set_path(output_db, 'Target_Addresses')
generate_target_addresses_csv(target_addresses_fc)
def extract(self):
"""Extracts data from a google spreadsheet.
Google spreadsheet url is contained in the config yaml.
Returns:
./data/addresses.csv with addresses from the google spreadsheet is created.
"""
# Get data
arcpy.AddMessage('Extracting addresses from google spreadsheet')
r = self.s.get(self.config_dict.get('gsheet_url'))
arcpy.AddMessage(f'HTTP Response: {r.status_code} \n {r.content} \n')
r.encoding = 'utf-8'
data = r.text
# Write data to csv
csv_path = set_path(self.config_dict['etl_dir'], 'addresses.csv')
arcpy.AddMessage(f'Writing data to addresses.csv in {csv_path}\n')
with open(csv_path, 'w', encoding='utf-8') as f:
f.write(data)
def load(self):
"""Loads new_addresss.csv into an arcpy feature class.
Returns:
arcpy XY table to point loads a feature class into an ArcGIS Pro db.
"""
arcpy.AddMessage('\nCreating a point feature class from input table\n')
# Setup local variables
in_table = set_path(self.config_dict['etl_dir'], 'new_addresses.csv')
out_feature_class = 'avoid_points'
x_coords = 'X'
y_coords = 'Y'
# Create XY event layer
arcpy.management.XYTableToPoint(in_table, out_feature_class, x_coords, y_coords)
arcpy.AddMessage(
f'\nFeature class avoid_points created with {arcpy.GetCount_management(out_feature_class)} rows.')
def run_model():
# Check setup
user_inputs = {'intersect_fc': 'IntersectAnalysis', 'buf_distance': '2500 Feet'}
output_db = config_dict.get('output_gdb_dir')
arcpy.AddMessage(f'output db: {output_db}')
aprx_path = set_path(config_dict.get('proj_dir'), 'WestNileOutbreak.aprx')
aprx = arcpy.mp.ArcGISProject(aprx_path)
arcpy.AddMessage(f'aprx path: {aprx.filePath}')
mp = get_map(aprx, 'Map')
# # Buffer Analysis
buf_fc_list = ['Mosquito_Larval_Sites', 'Wetlands_Regulatory', 'Lakes_and_Reservoirs', 'OSMP_Properties',
'avoid_points']
for fc in buf_fc_list:
buf_distance = user_inputs['buf_distance']
input_fc_name = fc
buf_fc_name = f'{fc}_buf'
buf_fc = set_path(output_db, buf_fc_name)
buffer(mp, input_fc_name, buf_fc, buf_fc_name, buf_distance)
aprx.save()
# Intersect Analysis
# for loop is used to create intersect_fc_list for intersect function (including paths to output_db)
intersect_fc_list = []
for fn in buf_fc_list:
if fn == 'avoid_points':
arcpy.AddMessage(
'\nSkipping avoid_points for Intersect Analysis they will be used for Symmetrical Difference.\n')
else:
intersect_fn = set_path(output_db, f'{fn}_buf')
intersect_fc_list.append(intersect_fn)
intersect_fc_name = user_inputs['intersect_fc']
inter = set_path(output_db, intersect_fc_name)
intersect(mp, intersect_fc_list, inter, intersect_fc_name)
aprx.save()
# Query by Location
join_output_name = 'IntersectAnalysis_Join_BoulderAddresses'
jofc = set_path(output_db, join_output_name)
sp = arcpy.SpatialJoin_analysis('Boulder_Addresses', inter, jofc, join_type="KEEP_COMMON", match_option="WITHIN")
check_status(sp)
# Record Count
record_count = arcpy.GetCount_management(jofc)
arcpy.AddMessage(f'\nBoulder Addresses at-risk = {record_count[0]}\n')
# Clip (Analysis)
# https://pro.arcgis.com/en/pro-app/latest/tool-reference/analysis/clip.htm
inFeatures = set_path(output_db, 'avoid_points_buf')
clipFeatures = set_path(output_db, 'IntersectAnalysis')
clipOutput = set_path(output_db, 'clip_intersect')
# Execute Clip
c = arcpy.Clip_analysis(inFeatures, clipFeatures, clipOutput)
check_status(c)
# Record re-count
join_output_name = 'clip_intersect_Join_BoulderAddresses'
jofc = set_path(output_db, join_output_name)
sp = arcpy.SpatialJoin_analysis('Boulder_Addresses', set_path(output_db, 'clip_intersect'), jofc,
join_type="KEEP_COMMON", match_option="WITHIN")
check_status(sp)
record_count = arcpy.GetCount_management(jofc)
arcpy.AddMessage(
f'\nBoulder Addresses in risk zone that need to be opted out of pesticide spraying = {record_count[0]}\n')
def run_analysis(output_db):
"""Analysis orchestration.
Coordinates geospatial analysis operations to create required output feature classes:
- final_analysis
- avoid_points_buf
- Target_Addresses
Args:
output_db: path of the output data base so each geoprocessing tool can write ouput feature classes.
Returns:
Results dictionary with the addresses at risk and map subtitle.
Side effect is final_analysis, avoid_points_buf, Target_Addresses exist in output_db
"""
# Start Input GUI
user_inputs = input_gui()
logger.info(f'Simulation Parameters: {user_inputs}')
# Buffer Analysis
# Create buffers around high risk areas that will require pesticide control spraying.
# Avoid points is also buffered here for convenience.
# Avoid points will not be included in the intersect analysis.
# Avoid points buffer represents individuals that signed up to opt out of pesticide control spraying.
buf_fc_list = [
'Mosquito_Larval_Sites', 'Wetlands_Regulatory', 'Lakes_and_Reservoirs',
'OSMP_Properties', 'avoid_points'
]
for fc in buf_fc_list:
buf_distance = user_inputs['buf_distance']
buf_fc_name = f'{fc}_buf'
buf_fc = set_path(output_db, buf_fc_name)
buffer(fc, buf_fc, buf_distance)
# Intersect Analysis
# Create an intersect feature layer of all the high risk buffer areas.
# The intersect feature layer represents the highest risk zone for West Nile Virus transmission.
# The intersect feature layer includes all areas that will require pesticide control spraying.
intersect_fc_list = []
for fn in buf_fc_list:
if fn == 'avoid_points':
arcpy.AddMessage(
'\nSkipping avoid_points not used for Intersect Analysis.\n')
else:
intersect_fn = set_path(output_db, f'{fn}_buf')
intersect_fc_list.append(intersect_fn)
intersect_fc_name = user_inputs['intersect_fc']
intersect_fc = set_path(output_db, intersect_fc_name)
intersect(intersect_fc_list, intersect_fc)
# Erase the intersection of the intersect layer and the avoid points.
# Pesticide control spraying needs to occur in the intersect layer.
# However the city can not spray in the avoid points buffer.
# Therefore the avoid points will be erased from the intersect layer.
# The resulting layer will be safe for pesticide control spraying.
erase_input = intersect_fc
erase_fc = set_path(output_db, 'avoid_points_buf')
erase_output = set_path(output_db, 'final_analysis')
erase(erase_input, erase_fc, erase_output)
# Perform a spatial join between Boulder_addresses and final_analysis.
# Then count the addresses in the Target_addresses layer.
# This count represents the addresses impacted by pesticide spraying.
# def spatial_join(target_fc, join_fc, output_fc):
# def record_count(count_fc):
target_fc = 'Boulder_Addresses'
join_fc = set_path(output_db, 'final_analysis')
join_output_fc = set_path(output_db, 'Target_Addresses')
spatial_join(target_fc, join_fc, join_output_fc)
addresses_at_risk_count = record_count(join_output_fc)
arcpy.AddMessage(
f'\nBoulder Addresses at-risk = {addresses_at_risk_count}\n')
# Create a dictionary of results for external use
results = {
'addresses_at_risk_count': addresses_at_risk_count,
'map_subtitle': user_inputs['map_subtitle']
}
return results
def run_model():
# setup the logger to generate log file use commands: logger.debug(msg), logger.info(msg)
setup_logging(level='DEBUG',
fn=f'{config_dict["proj_dir"]}/{config_dict["log_fn"]}')
# Start Input GUI
user_inputs = input_gui()
logger.info('Starting West Nile Virus Simulation')
logger.info(f'Simulation Parameters: {user_inputs}')
# setup arcpy environment
output_db = config_dict.get('output_gdb_dir')
arcpy.AddMessage(f'output db: {output_db}')
aprx_path = set_path(config_dict.get('proj_dir'), 'WestNileOutbreak.aprx')
aprx = arcpy.mp.ArcGISProject(aprx_path)
arcpy.AddMessage(f'aprx path: {aprx.filePath}')
mp = get_map(aprx, 'Map')
# Buffer Analysis
buf_fc_list = [
'Mosquito_Larval_Sites', 'Wetlands_Regulatory', 'Lakes_and_Reservoirs',
'OSMP_Properties', 'avoid_points'
]
for fc in buf_fc_list:
buf_distance = user_inputs['buf_distance']
input_fc_name = fc
buf_fc_name = f'{fc}_buf'
buf_fc = set_path(output_db, buf_fc_name)
buffer(mp, input_fc_name, buf_fc, buf_fc_name, buf_distance)
aprx.save()
# Intersect Analysis
# for loop is used to create intersect_fc_list for intersect function (including paths to output_db)
intersect_fc_list = []
for fn in buf_fc_list:
if fn == 'avoid_points':
arcpy.AddMessage(
'\nSkipping avoid_points for Intersect Analysis they will be used for Symmetrical Difference.\n'
)
else:
intersect_fn = set_path(output_db, f'{fn}_buf')
intersect_fc_list.append(intersect_fn)
intersect_fc_name = user_inputs['intersect_fc']
inter = set_path(output_db, intersect_fc_name)
intersect(mp, intersect_fc_list, inter, intersect_fc_name)
aprx.save()
# Query by Location
logger.debug('Starting Spatial Join geoprocessing.')
join_output_name = 'IntersectAnalysis_Join_BoulderAddresses'
jofc = set_path(output_db, join_output_name)
sp = arcpy.SpatialJoin_analysis('Boulder_Addresses',
inter,
jofc,
join_type="KEEP_COMMON",
match_option="WITHIN")
check_status(sp)
logger.debug('Spatial Join geoprocessing complete.')
# Record Count
logger.debug('Starting Get Count geoprocessing.')
record_count = arcpy.GetCount_management(jofc)
arcpy.AddMessage(f'\nBoulder Addresses at-risk = {record_count[0]}\n')
logger.debug('Get Count geoprocessing complete.')
# Clip (Analysis)
# https://pro.arcgis.com/en/pro-app/latest/tool-reference/analysis/clip.htm
logger.debug('Starting Clip geoprocessing.')
inFeatures = set_path(output_db, 'avoid_points_buf')
clipFeatures = set_path(output_db, user_inputs['intersect_fc'])
clipOutput = set_path(output_db, 'clip_intersect')
# Execute Clip
c = arcpy.Clip_analysis(inFeatures, clipFeatures, clipOutput)
check_status(c)
logger.debug('Clip geoprocessing complete.')
# Record re-count
logger.debug('Starting Spatial Join geoprocessing.')
join_output_name = 'clip_intersect_Join_BoulderAddresses'
jofc = set_path(output_db, join_output_name)
sp = arcpy.SpatialJoin_analysis('Boulder_Addresses',
set_path(output_db, 'clip_intersect'),
jofc,
join_type="KEEP_COMMON",
match_option="WITHIN")
check_status(sp)
logger.debug('Spatial Join geoprocessing complete.')
logger.debug('Starting Get Count geoprocessing.')
record_count = arcpy.GetCount_management(jofc)
logger.debug('Get Count geoprocessing complete.')
arcpy.AddMessage(
f'\nBoulder Addresses in risk zone that need to be opted out of pesticide spraying = {record_count[0]}\n'
)
# Add desired features to output map and colour the features
map_features = [(user_inputs['intersect_fc'], [255, 235, 190, 100]),
('avoid_points_buf', [115, 178, 255, 100]),
('clip_intersect_Join_BoulderAddresses',
[102, 119, 205, 100])]
for f, c in map_features:
fc_name = f
fc = set_path(output_db, f)
colour = c
add_feature_to_map(mp, fc_name, fc, colour)
aprx.save()
# Export final map
export_map(user_inputs['map_subtitle'])
aprx.save()