def merge (tile_file_names_table, fc_output):
# Chunk up the list of tiles
fns = [fn[0] for fn in tile_file_names.read_file_names(tile_file_names_table)]
fns_list = [fns[i:i+_CHUNK_SIZE] for i in range(0, len(fns), _CHUNK_SIZE)]
sr = arcpy.Describe(fns[0]).spatialReference
# Delete the output feature set
arcpy.Delete_management(fc_output)
arcpy.CreateFeatureclass_management(os.path.dirname(fc_output), os.path.basename(fc_output), "POLYGON", fns[0], '','', sr)
i=0
for fns in fns_list:
i += 1
common_functions.log_progress("Processing segment", len(fns_list), i)
merge_one_chunk (fns, sr, fc_output)
def compute(fc_input, zone_field, rasters, fc_output):
arcpy.CheckOutExtension("Spatial")
temporary_assets = list()
try:
# Create an index on the zone field
common_functions.create_index(fc_input, [zone_field], 'ZoneIdx')
# Delete the output feature set
arcpy.Delete_management(fc_output)
# Get a list of zone field value ranges (chunks). In addition to circumventing the documented
# 170000 record limitation, this also allows us to do the processing in memory
zone_val_chunks = get_zone_val_chunks(fc_input, zone_field)
# Prepare the in-memory feature classes
fc_in_int = os.path.join('in_memory', 'fc_int')
fc_out_int = os.path.join('in_memory', 'fc_out')
temporary_assets += [fc_in_int, fc_out_int]
i = 0
for zone_val_min, zone_val_max in zone_val_chunks:
i += 1
common_functions.log_progress("Processing segment",
len(zone_val_chunks), i)
# Create a feature class for this chunk
where_clause = '("' + zone_field + '">=' + str(
zone_val_min) + " AND " + '"' + zone_field + '"<=' + str(
zone_val_max) + ')'
arcpy.FeatureClassToFeatureClass_conversion(
fc_input, os.path.dirname(fc_in_int),
os.path.basename(fc_in_int), where_clause)
# Compute the zonal stats for this chunk
arcpy.sa.ZonalStatisticsAsTable(fc_in_int, zone_field, rasters,
fc_out_int)
# Append results to the output feature class
if arcpy.Exists(fc_output):
arcpy.Append_management(fc_out_int, fc_output)
else:
arcpy.CopyRows_management(fc_out_int, fc_output)
finally:
# Clean up
arcpy.CheckInExtension("Spatial")
for temporary_asset in temporary_assets:
log('Deleting ' + temporary_asset)
arcpy.Delete_management(temporary_asset)
log("Done")
def join(left_side_fc, left_side_key, right_side_fc, right_side_key,
right_side_include_fields):
log('Join left side: ' + left_side_fc)
log('Join right side: ' + right_side_fc)
log('Join keys: ' + str(left_side_key) + ':' + str(right_side_key))
common_functions.create_index(left_side_fc, [left_side_key], 'LeftIdx')
common_functions.create_index(right_side_fc, [right_side_key], 'RightIdx')
# Update the left side feature class with the fields from the right side (they will be populated in the next step)
left_side_add_fields(left_side_fc, [
f for f in arcpy.ListFields(right_side_fc)
if f.name in right_side_include_fields.split(';')
])
# Prepare to write values to left side
right_side_cursor = right_side_row_gen(
right_side_fc, [right_side_key] + right_side_include_fields.split(';'),
right_side_key)
right_side_row = right_side_cursor.next()
# Since both cursors return rows sorted, we simply advance then in tandem. When we find a matching
# key, we simply copy to specified right hand fields into the left side feature class
# the matching keys
count = int(arcpy.GetCount_management(left_side_fc).getOutput(0))
arcpy.SetProgressor("step", "Joining files...", 0, count, 1)
i = 0
with arcpy.da.UpdateCursor(
left_side_fc,
[left_side_key] + right_side_include_fields.split(';'),
sql_clause=(None,
'ORDER BY ' + left_side_key)) as left_side_cursor:
for left_side_row in left_side_cursor:
i += 1
common_functions.log_progress("Joining record ", count, i)
try:
while left_side_row[0] > right_side_row[0]:
right_side_row = right_side_cursor.next()
if left_side_row[0] == right_side_row[0]:
left_side_cursor.updateRow(right_side_row)
except StopIteration:
arcpy.AddWarning('End of join table.')
break
del left_side_cursor
log('Done.')
def populate (tuple_list):
if len(tuple_list) > sys.maxint and 'UNIQUE_ID' not in [i[_FIELD_VALUE] for i in tuple_list]: # turn off mp support due to ESRI bug
# if len(tuple_list) > 1 and 'UNIQUE_ID' not in [i[_FIELD_VALUE] for i in tuple_list]: # turn off mp support due to ESRI bug in AddField
# Use multiprocessing support to do the work
multiprocessing.set_executable(os.path.join(common_functions.get_install_path(), 'pythonw.exe'))
log('Launching ' + str(_threads) + ' worker processes')
tuple_lists = [tuple_list[i::_threads] for i in xrange(_threads if _threads < len(tuple_list) else len(tuple_list)) ]
p = multiprocessing.Pool(_threads)
p.map(partial(populate_mp, scratch_ws=arcpy.env.scratchWorkspace), tuple_lists)
p.close()
else:
fc_count = 1
for tuple_ in tuple_list:
fc = tuple_[_FC]
field_name = tuple_[_FIELD_NAME]
field_value = tuple_[_FIELD_VALUE]
field_type = tuple_[_FIELD_TYPE]
common_functions.log_progress ('Populating ' + field_name + ' with ' + str(field_value) + ' in ' + fc, len(tuple_list), fc_count)
populate_fc (fc, field_name, field_value, field_type)
fc_count = fc_count+1
def log_progress (message, max_range, step_count, threads=1):
common_functions.log_progress (message, max_range, step_count, threads)