def replace_null_values(fc,
fields="*",
oid_field=None,
null_value=0,
where_clause=None):
"""updates a set of rows in chunks
"""
if fields is None or \
isinstance(fields, list) == False or \
fields == "*":
fields = [field.name for field in arcpy.ListFields(fc) \
if field.type not in ('Geometry', 'Blob', 'Raster')]
if oid_field is None:
oid_field = arcpy.Describe(fc).OIDFieldName
chunk_size = calc_chunk_size()
if oid_field not in fields:
fields.append(oid_field)
with da.SearchCursor(fc, fields, where_clause=where_clause) as cursor:
search_fields = [
field for field in cursor.fields if field != oid_field
]
for group in grouper_it(chunk_size, cursor):
df = pd.DataFrame.from_records(group, columns=cursor.fields)
for field in search_fields:
df.loc[df[field].isnull(), field] = null_value
del field
array = df.to_records(index=False)
da.ExtendTable(fc, oid_field, array, oid_field, False)
del array
return fc
def extend_table(table, rows=None):
"""
Adds the required columns to the table and appends new records if
given.
"""
try:
if rows is None:
rows = []
dtypes = np.dtype([('_ID', np.int), ('MEAN', np.float64),
('MEDIAN', np.float64), ('MODE', np.float64),
('MIN', np.float64), ('MAX', np.float64),
('NO_DATE_CNT', np.int32),
('NO_DATE_PCT', np.float64),
('FEATURE_CNT', np.int32), ('PA_SCORE', np.int32),
("TIER", '|S1024')])
array = np.array(rows, dtypes)
da.ExtendTable(table, "OID@", array, "_ID", False)
except:
line, filename, synerror = trace()
raise FunctionError({
"function": "",
"line": line,
"filename": filename,
"synerror": synerror,
"arc": str(arcpy.GetMessages(2))
})
def extend_table(rows, table):
"""
appends the results of the array to the existing table by an objectid
"""
try:
dtypes = np.dtype([
('_ID', np.int), ('DOM_DATE', '|S48'), ('DOM_DATE_CNT', np.int32),
('DOM_DATE_PER', np.float64), ('DOM_YEAR', np.int32),
('DOM_YEAR_CNT', np.int32), ('DOM_YEAR_PER', np.float64),
('OLDEST_DATE', '|S1024'), ('NEWEST_DATE', '|S1024'),
('NO_DATE_CNT', np.int32), ('NO_DATE_PER', np.float64),
('PCT_2_YEAR', np.float64), ('PCT_5_YEAR', np.float64),
('PCT_10_YEAR', np.float64), ('PCT_15_YEAR', np.float64),
('PCT_15_PLUS_YEAR', np.float64), ('FEATURE_CNT', np.int32),
('CURRENCY_SCORE', np.int32)
])
array = np.array(rows, dtypes)
da.ExtendTable(table, "OID@", array, "_ID", False)
return table
except:
line, filename, synerror = trace()
raise FunctionError({
"function": "",
"line": line,
"filename": filename,
"synerror": synerror,
"arc": str(arcpy.GetMessages(2))
})
def grid_fields(grid):
"""
Adds fields to a given polygon grid.
"""
try:
dtypes = np.dtype([
('_ID', np.int), ('DOM_DATE', '|S48'), ('DOM_DATE_CNT', np.int32),
('DOM_DATE_PER', np.float64), ('DOM_YEAR', np.int32),
('DOM_YEAR_CNT', np.int32), ('DOM_YEAR_PER', np.float64),
('OLDEST_DATE', '|S1024'), ('NEWEST_DATE', '|S1024'),
('NO_DATE_CNT', np.int32), ('NO_DATE_PER', np.float64),
('PCT_2_YEAR', np.float64), ('PCT_5_YEAR', np.float64),
('PCT_10_YEAR', np.float64), ('PCT_15_YEAR', np.float64),
('PCT_15_PLUS_YEAR', np.float64), ('FEATURE_CNT', np.int32),
('CURRENCY_SCORE', np.int32)
])
array = np.array([], dtypes)
da.ExtendTable(grid, "OID@", array, "_ID", False)
return grid
except:
line, filename, synerror = trace()
raise FunctionError({
"function": "grid_fields",
"line": line,
"filename": filename,
"synerror": synerror,
"arc": str(arcpy.GetMessages(2))
})
def extend_table(table, rows=None):
"""
Adds the required columns to the table and appends new records if
given.
"""
try:
if rows is None:
rows = []
dtypes = np.dtype(
[
('_ID', np.int),
('DOM_SCALE', np.float64),
('DOM_COUNT', np.int32),
('DOM_PER', np.float64),
('MIN_SCALE', np.float64),
('MIN_PER', np.float64),
('MAX_SCALE', np.float64),
('MAX_PER', np.float64),
('CNT_2500', np.int32),
('CNT_5000', np.int32),
('CNT_12500', np.int32),
('CNT_25000', np.int32),
('CNT_50000', np.int32),
('CNT_100000', np.int32),
('CNT_250000', np.int32),
('CNT_500000', np.int32),
('CNT_1000000', np.int32),
('PER_2500', np.float64),
('PER_5000', np.float64),
('PER_12500', np.float64),
('PER_25000', np.float64),
('PER_50000', np.float64),
('PER_100000', np.float64),
('PER_250000', np.float64),
('PER_500000', np.float64),
('PER_1000000', np.float64),
('COUNT', np.int32),
('MISSION_PLANNING', '|S1024'),
('POPULATION_SCALE', '|S1024'),
('THEM_ACC_SCORE', np.float64)
]
)
array = np.array(rows, dtypes)
da.ExtendTable(table, "OID@", array, "_ID", False)
return table
except:
line, filename, synerror = trace()
raise FunctionError(
{
"function": "extend_table",
"line": line,
"filename": filename,
"synerror": synerror,
"arc" : str(arcpy.GetMessages(2))
}
)
def insert_new_results(selected_auth_fl, selected_res_fl, authoritative_fc,
schema):
selected_auth_sdf = SpatialDataFrame.from_featureclass(selected_auth_fl)
print(len(selected_auth_sdf))
selected_res_sdf = SpatialDataFrame.from_featureclass(selected_res_fl)
print(len(selected_res_sdf))
fields = field_schema.get(schema)
#for f in fields:
# print(f)
# Write this function
dtypes = dts.get(schema)
fields = field_schema.get(schema)
for idx, sel_auth_row in enumerate(selected_auth_sdf.iterrows()):
geom = sel_auth_row[1].SHAPE.buffer(-.01)
oid = sel_auth_row[1].OBJECTID
# print(oid)
ext = geom.extent
sq = selected_res_sdf['SHAPE'].disjoint(geom) == False
df_current = selected_res_sdf[sq].copy()
df_current.reset_index(inplace=True)
#print(df_current.head())
if len(df_current) > 0:
# print("Here")
#['MEAN_CE90']
for f in fields:
try:
cur_val = df_current.loc[0].at[f]
#print(cur_val)
selected_auth_sdf.at[idx, f] = cur_val
except:
# break
print("Field doesn't exist")
insert_df = selected_auth_sdf.drop(['SHAPE'], axis=1, inplace=False)
records = insert_df.to_records(index=False)
rows = np.array(records, dtype=dtypes)
array = rows # np.array(rows, dtypes)
da.ExtendTable(authoritative_fc, "OID@", array, "_ID", False)
return authoritative_fc
def replace_values(fc,
fields="*",
oid_field=None,
find_value=None,
replace_value=0,
where_clause=None):
"""updates a set of rows in chunks
"""
try:
if fields is None or \
isinstance(fields, list) == False or \
fields == "*":
fields = [field.name for field in arcpy.ListFields(fc) \
if field.type not in ('Geometry', 'Blob', 'Raster')]
if oid_field is None:
oid_field = arcpy.Describe(fc).OIDFieldName
chunk_size = calc_chunk_size()
if oid_field not in fields:
fields.append(oid_field)
with da.SearchCursor(fc, fields, where_clause=where_clause) as cursor:
search_fields = [
field for field in cursor.fields if field != oid_field
]
for group in grouper_it(chunk_size, cursor):
df = pd.DataFrame.from_records(group, columns=cursor.fields)
for field in search_fields:
if find_value is None or \
str(find_value).lower() == "none" or \
str(find_value).lower().strip() == "":
df.loc[df[field].isnull(), field] = replace_value
else:
df.loc[df[field] == find_value, field] = replace_value
del field
array = df.to_records(index=False)
da.ExtendTable(fc, oid_field, array, oid_field, False)
del array
del df
return fc
except:
line, filename, synerror = trace()
raise FunctionError({
"function": "replace_values",
"line": line,
"filename": filename,
"synerror": synerror,
"arc": str(arcpy.GetMessages(2))
})
def extend_table(fc, array):
"""
extends to append field and data to an existing table
"""
try:
da.ExtendTable(fc, "OID@", array, "_ID", False)
return fc
except:
line, filename, synerror = trace()
raise FunctionError({
"function": "extend_table",
"line": line,
"filename": filename,
"synerror": synerror,
"arc": str(arcpy.GetMessages(2))
})
def create_error_fc(outFC, geometryType, sr=None):
"""
Creates an error feature class for a given geometry type and
spatial reference.
Output:
path to a feature class
"""
try:
if arcpy.Exists(outFC):
arcpy.AddMessage("Error Feature Class Already Exists, Recreating...")
arcpy.Delete_management(outFC)
arcpy.CreateFeatureclass_management(out_path=os.path.dirname(outFC),
out_name=os.path.basename(outFC),
geometry_type=geometryType.upper(),
spatial_reference=sr)
narray = np.array([],
np.dtype([('_ID', np.int),# _ID will not be included in the extend
('DEFICIENCY', '|S1024'),
('FEATURE_CLASS', '|S254'),
('SUBTYPE', '|S254'),
('ORIG_OID', np.int),
('DEFICIENCY_CNT', np.int)
])
)
da.ExtendTable(outFC,
"OID@",
narray,
"_ID")
return outFC
except:
line, filename, synerror = trace()
raise FunctionError(
{
"function": "",
"line": line,
"filename": __file__,
"synerror": synerror,
"arc" : str(arcpy.GetMessages(2))
}
)
def extend_table(table, rows=None):
"""
Adds the required columns to the table and appends new records if
given.
"""
try:
if rows is None:
rows = []
dtypes = np.dtype([
('_ID', np.int),
('MEAN_DEF_CNT', np.int32),
('MEDIAN_DEF_CNT', np.float64),
('MIN_DEF_CNT', np.int32),
('MAX_DEF_CNT', np.int32),
#STandard deviation
('PRI_NUM_DEF', np.int32),
('SEC_NUM_DEF', np.int32),
('PER_PRI', np.float64),
('PER_SEC', np.float64),
("PRI_ATTR_DEF", '|S20'), # pri_attr
("SEC_ATTR_DEF", '|S20'),
('PRI_ATTR_DEF_PER', np.float64),
('SEC_ATTR_DEF_PER', np.float64),
('FEATURE_CNT', np.int32),
('PRI_ATTR_DEF_CNT', np.float64),
('SEC_ATTR_DEF_CNT', np.float64),
('LC_SCORE', np.int32)
])
array = np.array(rows, dtypes)
da.ExtendTable(table, "OID@", array, "_ID", False)
return table
except:
line, filename, synerror = trace()
raise FunctionError({
"function": "extend_table",
"line": line,
"filename": filename,
"synerror": synerror,
"arc": str(arcpy.GetMessages(2))
})
def extend_table(table, rows=None):
"""
Adds the required columns to the table and appends new records if
given.
"""
try:
if rows is None:
rows = []
dtypes = np.dtype([('_ID', np.int), ('TDS_DENSITY', np.float64),
('COMP_DENSITY', np.float64),
('COMPLETENESS_VALUE', np.float64),
('DIFFERENCE', np.float64)])
array = np.array(rows, dtypes)
da.ExtendTable(table, "OID@", array, "_ID", False)
return table
except:
line, filename, synerror = trace()
raise FunctionError({
"function": "extend_table",
"line": line,
"filename": filename,
"synerror": synerror,
"arc": str(arcpy.GetMessages(2))
})
def data_comparison(in_grid,
in_fcs,
in_old_gdb,
in_new_gdb,
out_grid,
geom_type="POINT",
scratchGDB=env.scratchGDB):
"""
Generates rankings based on a given grid and data type
Inputs:
in_grid: path to area of interests
in_fcs: list of feature class names
in_old_gdb: old FGDB path for comparison
in_new_gdb: new FGDB path for comparison
out_grid: path of the output GDB feature class
geom_type: string value of POINT, POLYLINE, or POLYGON
"""
try:
merged_points_n = os.path.join("in_memory",
"merged_pts_n") #scratchGDB
merged_points_o = os.path.join("in_memory",
"merged_pts_o") #scratchGDB
temp_out_grid = os.path.join(scratchGDB, "grid")
old_stats = os.path.join(scratchGDB, "old_stats")
new_stats = os.path.join(scratchGDB, "new_stats")
# Copy Grid to Temp Folder
temp_out_grid = arcpy.CopyFeatures_management(in_grid,
temp_out_grid)[0]
# Merge all fcs into one fc
merged_points_n, total_n_count = merge_fcs(in_fcs,
merged_points_n,
gdb=in_new_gdb)
merged_points_o, total_o_count = merge_fcs(in_fcs,
merged_points_o,
gdb=in_old_gdb)
# intersect the grid
new_pts_int = arcpy.Intersect_analysis(
in_features=[merged_points_n, temp_out_grid],
out_feature_class=os.path.join(scratchGDB, "new_pts_int"),
join_attributes="ONLY_FID")[0]
old_pts_int = arcpy.Intersect_analysis(
in_features=[merged_points_o, temp_out_grid],
out_feature_class=os.path.join(scratchGDB, "old_pts_int"),
join_attributes="ONLY_FID")[0]
if geom_type.lower() == "point":
stat_fields_new = "FID_grid COUNT"
stat_fields_old = "FID_grid COUNT"
method = ["POINT"]
elif geom_type.lower() in ("polyline", 'polygon'):
arcpy.AddField_management(old_pts_int, "OLD_LENGTH", "FLOAT")
arcpy.CalculateField_management(
in_table=old_pts_int,
field="OLD_LENGTH",
expression="!shape.length@kilometers!",
expression_type="PYTHON_9.3",
code_block="")
arcpy.AddField_management(new_pts_int, "NEW_LENGTH", "FLOAT")
arcpy.CalculateField_management(
in_table=new_pts_int,
field="NEW_LENGTH",
expression="!shape.length@kilometers!",
expression_type="PYTHON_9.3",
code_block="")
if geom_type.lower() == "polygon":
arcpy.AddField_management(old_pts_int, "OLD_AREA", "FLOAT")
#!shape.area@squarekilometers!
arcpy.CalculateField_management(
in_table=old_pts_int,
field="OLD_AREA",
expression="!shape.area@squarekilometers!",
expression_type="PYTHON_9.3",
code_block="")
arcpy.AddField_management(new_pts_int, "NEW_AREA", "FLOAT")
arcpy.CalculateField_management(
in_table=new_pts_int,
field="NEW_AREA",
expression="!shape.area@squarekilometers!",
expression_type="PYTHON_9.3",
code_block="")
if geom_type.lower() == "polygon":
stat_fields_new = "FID_grid COUNT;NEW_LENGTH SUM;NEW_AREA SUM"
stat_fields_old = "FID_grid COUNT;OLD_LENGTH SUM;OLD_AREA SUM"
method = ['POINT', 'POLYLINE', 'POLYGON']
else:
stat_fields_new = "FID_grid COUNT;NEW_LENGTH SUM"
stat_fields_old = "FID_grid COUNT;OLD_LENGTH SUM"
method = ['POINT', 'POLYLINE']
# get the counts
old_stats = arcpy.Statistics_analysis(
in_table=old_pts_int,
out_table=old_stats,
statistics_fields=stat_fields_old,
case_field="FID_grid")[0]
new_stats = arcpy.Statistics_analysis(
in_table=new_pts_int,
out_table=new_stats,
statistics_fields=stat_fields_new,
case_field="FID_grid")[0]
# join the old stats to the new stats
if geom_type.lower() == "polygon":
arcpy.AlterField_management(new_stats,
field="SUM_NEW_LENGTH",
new_field_name="NEW_LENGTH")
arcpy.AlterField_management(new_stats,
field="SUM_NEW_AREA",
new_field_name="NEW_AREA")
out_fields = [
'FID_grid', 'FREQUENCY', 'SUM_OLD_LENGTH', 'SUM_OLD_AREA'
]
ndt = np.dtype([('FID_grid', '<i4'), ('OLD_FREQUENCY', '<i4'),
('OLD_LENGTH', np.float64),
('OLD_AREA', np.float64)])
export_fields = [
'FID_grid', 'FREQUENCY', 'OLD_FREQUENCY', 'OLD_LENGTH',
'NEW_LENGTH', 'OLD_AREA', 'NEW_AREA', 'SCORE', 'RANKING'
]
elif geom_type.lower() == "point":
out_fields = ['FID_grid', 'FREQUENCY']
ndt = np.dtype([('FID_grid', '<i4'), ('OLD_FREQUENCY', '<i4')])
export_fields = [
'FID_grid', 'FREQUENCY', 'OLD_FREQUENCY', 'SCORE', 'RANKING'
]
elif geom_type.lower() == "polyline":
arcpy.AlterField_management(new_stats,
field="SUM_NEW_LENGTH",
new_field_name="NEW_LENGTH")
out_fields = ['FID_grid', 'FREQUENCY', 'SUM_OLD_LENGTH']
ndt = np.dtype([('FID_grid', '<i4'), ('OLD_FREQUENCY', '<i4'),
('OLD_LENGTH', np.float64)])
export_fields = [
'FID_grid', 'FREQUENCY', 'OLD_FREQUENCY', 'OLD_LENGTH',
'NEW_LENGTH', 'SCORE', 'RANKING'
]
old_array = da.TableToNumPyArray(in_table=old_stats,
field_names=out_fields)
old_array.dtype = ndt
# Add SCORE and RANKING fields and remove unneeded fields
da.ExtendTable(new_stats, "FID_grid", old_array, "FID_grid", False)
array = np.array([],
np.dtype([('_id', np.int32), ('SCORE', np.float64),
('RANKING', np.int64)]))
da.ExtendTable(new_stats,
arcpy.Describe(new_stats).OIDFieldName, array, "_id",
False)
arcpy.DeleteField_management(new_stats, ['COUNT_FID_grid'])
array = da.TableToNumPyArray(in_table=new_stats,
field_names=export_fields,
null_value=0)
# Calculate the rankings
tcsv, column_list = calculate_frequency_ranking(array=array,
methods=method)
array = da.TableToNumPyArray(tcsv, column_list)
da.ExtendTable(temp_out_grid,
arcpy.Describe(temp_out_grid).OIDFieldName, array,
"FID_grid")
# Clean up NULL values
if geom_type.lower() == "point":
sql = """RANKING IS NULL"""
fields = ['RANKING']
elif geom_type.lower() == "polyline":
sql = """RANKING IS NULL OR RANKING_LENGTH IS NULL"""
fields = ['RANKING', 'RANKING_LENGTH']
elif geom_type.lower() == "polygon":
sql = """RANKING IS NULL OR RANKING_LENGTH IS NULL OR RANKING_AREA IS NULL"""
fields = ['RANKING', 'RANKING_LENGTH', 'RANKING_AREA']
with da.UpdateCursor(temp_out_grid, fields, where_clause=sql) as urows:
for row in urows:
if geom_type.lower() == "point":
row[0] = 0
elif geom_type.lower() == "polyline":
if row[0] is None:
row[0] = 0
if row[1] is None:
row[1] = 0
elif geom_type.lower() == "polygon":
if row[0] is None:
row[0] = 0
if row[1] is None:
row[1] = 0
if row[2] is None:
row[2] = 0
urows.updateRow(row)
del row
del urows
# return the output grid
return arcpy.CopyFeatures_management(temp_out_grid, out_grid)[0]
except:
line, filename, synerror = trace()
raise FunctionError({
"function": "data_comparison",
"line": line,
"filename": filename,
"synerror": synerror,
"arc": str(arcpy.GetMessages(2))
})
def main(*argv):
""" main driver of program """
try:
fc = argv[0]
find_value = argv[1] #None #
fields = str(argv[2]).split(';') #str("name;name_en").split(";") #
copy_fc = argv[3]
feat_count = arcpy.GetCount_management(fc)
index = int(feat_count.getOutput(0)) - 1
text_fields = ",".join(fields)
flag_field = "FLAGGED_DATA"
oid_field = None
scratchGDB = env.scratchGDB
#copy_fc = os.path.join(scratchGDB, os.path.basename(fc))
isnumber = False
# Logic
#
if find_value is None or \
find_value == "" or \
str(find_value).lower() == "none":
find_value = None
elif find_value.startswith("-") and \
find_value.find(".") < -1 and \
find_value[1:].isdigit():
find_value = int(find_value)
isnumber = True
elif find_value.find(".") < -1 and \
find_value.isdigit():
find_value = int(find_value)
isnumber = True
if isnumber == False:
try:
find_value = float(find_value)
isnumber = True
except:
isnumber = False
if arcpy.Exists(copy_fc):
arcpy.Delete_management(copy_fc)
copy_fc = copy_rows(
source=fc, destination=copy_fc) # Instead of arcpy.CopyFeatures()
#arcpy.CopyFeatures_management(fc, copy_fc)
#if check_shape:
# fields.append("SHAPE@")
if oid_field is None:
oid_field = arcpy.Describe(fc).OIDFieldName
if oid_field not in fields:
fields.append(oid_field)
chunk_size = calc_chunk_size()
count = 0
dtypes = np.dtype([('_ID', np.int), ('NULL_COLUMNS', '|S255'),
('FLAG_FIELDS', '|S255'), ('NULL_COUNT', np.int32),
('FLAGGED_DATA', np.int32)])
#('SELECTED_FIELDS', '|S1024'),
#('FLAGGED_FIELDS', '|S1024'),
with da.SearchCursor(copy_fc, fields, where_clause=None) as cursor:
search_fields = [
field for field in cursor.fields if field != oid_field
]
for group in grouper_it(chunk_size, cursor):
flagged_fields = []
count += 1
df = pd.DataFrame.from_records(group, columns=cursor.fields)
df['NULL_COLUMNS'] = df.apply(
lambda x: ','.join(x[x.isin([find_value])].index), axis=1)
#df.apply(lambda x: ','.join(x[x.isnull()].index),axis=1)#
df['FLAG_FIELDS'] = text_fields
#df['SELECTED_FIELDS'] = text_fields
#df['FLAGGED_FIELDS'] = text_fields
if find_value is None:
df['NULL_COUNT'] = len(df.columns) - df.count(axis=1)
elif find_value and isnumber:
df['NULL_COUNT'] = (df[fields] == find_value).sum(axis=1)
elif find_value and isnumber == False:
df['NULL_COUNT'] = (df[fields] == find_value).sum(axis=1)
df['NULL_COUNT'] = df[fields].isin([find_value
]).sum(axis=1)
df[flag_field] = 0
df.loc[df['NULL_COUNT'] > 0, flag_field] = 1
for fld in search_fields:
del df[fld]
array = df.to_records(index=False).tolist()
array = np.array(array, dtypes)
da.ExtendTable(copy_fc, oid_field, array, '_ID', False)
del group
del df
del array
#arcpy.SetParameterAsText(3, copy_fc)
except arcpy.ExecuteError:
line, filename, synerror = trace()
arcpy.AddError("error on line: %s" % line)
arcpy.AddError("error in file name: %s" % filename)
arcpy.AddError("with error message: %s" % synerror)
arcpy.AddError("ArcPy Error Message: %s" % arcpy.GetMessages(2))
except FunctionError as f_e:
messages = f_e.args[0]
arcpy.AddError("error in function: %s" % messages["function"])
arcpy.AddError("error on line: %s" % messages["line"])
arcpy.AddError("error in file name: %s" % messages["filename"])
arcpy.AddError("with error message: %s" % messages["synerror"])
arcpy.AddError("ArcPy Error Message: %s" % messages["arc"])
except:
line, filename, synerror = trace()
arcpy.AddError("error on line: %s" % line)
arcpy.AddError("error in file name: %s" % filename)
arcpy.AddError("with error message: %s" % synerror)
def main(*argv):
""" main driver of program """
try:
old_fc = argv[0]
new_fc = argv[1]
polygon_grid = argv[2]
fields = str(argv[3]).split(';') #argv[3]
out_gdb = argv[4]
# Local Variables
#
scratchGDB = env.scratchGDB
scratchFolder = env.scratchFolder
copy_grid = os.path.join(out_gdb, "grid")
copy_old_fc = os.path.join(scratchGDB,
os.path.basename(old_fc) + "_old")
copy_new_fc = os.path.join(scratchGDB,
os.path.basename(new_fc) + "_new")
intOld = os.path.join(scratchGDB, "intold")
intNew = os.path.join(scratchGDB, "intnew")
sumOld = os.path.join(scratchGDB, "sum_old")
sumNew = os.path.join(scratchGDB, "sum_new")
temp_csv = os.path.join(scratchFolder, "temp_csv.csv")
# Logic
#
# Validate that fields exist in both tables
old_fc_flds = {
field.name
for field in arcpy.ListFields(old_fc) if field.name in fields
}
new_fc_flds = {
field.name
for field in arcpy.ListFields(new_fc) if field.name in fields
}
if len(new_fc_flds) != len(old_fc_flds):
missing = list(old_fc_flds - new_fc_flds) + list(new_fc_flds -
old_fc_flds)
arcpy.AddWarning("Comparison Fields Missing: %s" %
",".join(missing))
fields = [fields.remove(v) for v in missing if v in fields]
if len(fields) == 0:
raise Exception(
"All fields provided do not exist in each dataset. Nothing to compare."
)
#copy the data
copy_grid = arcpy.CopyFeatures_management(polygon_grid, copy_grid)[0]
copy_old_fc = arcpy.FeatureClassToFeatureClass_conversion(
old_fc,
out_path=os.path.dirname(copy_old_fc),
out_name=os.path.basename(copy_old_fc))[0]
copy_new_fc = arcpy.FeatureClassToFeatureClass_conversion(
new_fc,
out_path=os.path.dirname(copy_new_fc),
out_name=os.path.basename(copy_new_fc))[0]
# get the null counts
oldResult = calculate_nulls(copy_old_fc, fields)
newResult = calculate_nulls(copy_new_fc, fields)
# Intersect Polygon with old/new
intOld = arcpy.Intersect_analysis(in_features=[copy_grid, copy_old_fc],
out_feature_class=intOld,
join_attributes="ALL",
cluster_tolerance="-1 Unknown",
output_type="INPUT")[0]
intNew = arcpy.Intersect_analysis(in_features=[copy_grid, copy_new_fc],
out_feature_class=intNew,
join_attributes="ALL",
cluster_tolerance="-1 Unknown",
output_type="INPUT")[0]
# Aggregavte and Average out ranking by grid OID
case_field = "FID_%s" % os.path.basename(copy_grid)
sumOld = arcpy.Statistics_analysis(
in_table=intOld,
out_table=sumOld,
statistics_fields="NULL_COUNT SUM;PERCENT_COMP MEAN;RANKING MEAN",
case_field=case_field)[0]
sumNew = arcpy.Statistics_analysis(
in_table=intNew,
out_table=sumNew,
statistics_fields="NULL_COUNT SUM;PERCENT_COMP MEAN;RANKING MEAN",
case_field=case_field)[0]
array_old = da.TableToNumPyArray(sumOld, [
case_field, 'SUM_NULL_COUNT', 'MEAN_PERCENT_COMP', 'MEAN_RANKING'
])
dtype_old = [('FID_grid', '<i4'), ('NULL_COUNT_OLD', '<f8'),
('PERCENT_COMP_OLD', '<f8'), ('RANKING_OLD', '<f8')]
array_old.dtype = dtype_old
array_new = da.TableToNumPyArray(sumNew, [
case_field, 'SUM_NULL_COUNT', 'MEAN_PERCENT_COMP', 'MEAN_RANKING'
])
dtype_new = [('FID_grid', '<i4'), ('NULL_COUNT_NEW', '<f8'),
('PERCENT_COMP_NEW', '<f8'), ('RANKING_NEW', '<f8')]
array_new.dtype = dtype_new
# Join stats table back to grid
df_old = pd.DataFrame(data=array_old,
columns=array_old.dtype.fields.keys())
df_new = pd.DataFrame(data=array_new,
columns=array_new.dtype.fields.keys())
join_df = df_new.set_index('FID_grid').join(
df_old.set_index('FID_grid'))
join_df['FID_grid'] = join_df.index
dtype_join = list(set(dtype_new + dtype_old))
join_df.loc[(join_df['RANKING_OLD'].isnull()), 'RANKING_OLD'] = 0
join_df.loc[(join_df['RANKING_NEW'].isnull()), 'RANKING_NEW'] = 0
join_df[
'DIFF_RANKING'] = join_df['RANKING_NEW'] - join_df['RANKING_OLD']
if os.path.isfile(temp_csv):
os.remove(temp_csv)
join_df.to_csv(
temp_csv,
columns=[case_field, 'RANKING_OLD', 'RANKING_NEW', 'DIFF_RANKING'])
array = da.TableToNumPyArray(
temp_csv,
[case_field, 'RANKING_OLD', 'RANKING_NEW', 'DIFF_RANKING'])
oid = arcpy.Describe(copy_grid).OIDFieldName
da.ExtendTable(copy_grid, oid, array, case_field, False)
with arcpy.da.UpdateCursor(
copy_grid, ['DIFF_RANKING', 'RANKING_OLD', 'RANKING_NEW'],
where_clause=
"DIFF_RANKING IS NULL or RANKING_OLD is NULL or RANKING_NEW is NULL"
) as urows:
for urow in urows:
if urow[0] is None:
urow[0] = 0
if urow[1] is None:
urow[1] = 0
if urow[2] is None:
urow[2] = 0
#urow[0] = 0
urows.updateRow(urow)
del urow
del urows
if os.path.isfile(temp_csv):
os.remove(temp_csv)
# Output
#
arcpy.SetParameterAsText(5, copy_grid)
except arcpy.ExecuteError:
line, filename, synerror = trace()
arcpy.AddError("error on line: %s" % line)
arcpy.AddError("error in file name: %s" % filename)
arcpy.AddError("with error message: %s" % synerror)
arcpy.AddError("ArcPy Error Message: %s" % arcpy.GetMessages(2))
except FunctionError as f_e:
messages = f_e.args[0]
arcpy.AddError("error in function: %s" % messages["function"])
arcpy.AddError("error on line: %s" % messages["line"])
arcpy.AddError("error in file name: %s" % messages["filename"])
arcpy.AddError("with error message: %s" % messages["synerror"])
arcpy.AddError("ArcPy Error Message: %s" % messages["arc"])
except:
line, filename, synerror = trace()
arcpy.AddError("error on line: %s" % line)
arcpy.AddError("error in file name: %s" % filename)
arcpy.AddError("with error message: %s" % synerror)
def _arcpy_to_featureclass(df, out_name, out_location=None,
overwrite=True, out_sr=None,
skip_invalid=True):
"""
"""
import arcgis
import numpy as np
import datetime
from arcpy import da
from arcgis.features import SpatialDataFrame
gtype = df.geometry_type.upper()
gname = df.geometry.name
df = df.copy()
if overwrite and \
arcpy.Exists(os.path.join(out_location, out_name)):
arcpy.Delete_management(os.path.join(out_location, out_name))
elif overwrite == False and \
arcpy.Exists(os.path.join(out_location, out_name)):
raise Exception("Dataset exists, please provide a new out_name or location.")
if out_sr is None:
try:
if isinstance(df.sr, dict):
sr = arcgis.geometry.SpatialReference(df.sr).as_arcpy
elif isinstance(df.sr, arcgis.geometry.SpatialReference):
sr = df.sr.as_arcpy
except:
sr = arcpy.SpatialReference(4326)
else:
if isinstance(df.sr, dict):
sr = arcgis.geometry.SpatialReference(df.sr).as_arcpy
elif isinstance(df.sr, arcgis.geometry.SpatialReference):
sr = df.sr.as_arcpy
elif isinstance(out_sr, arcpy.SpatialReference):
sr = out_sr
fc = arcpy.CreateFeatureclass_management(out_path=out_location,
out_name=out_name,
geometry_type=gtype.upper(),
spatial_reference=sr)[0]
df['JOIN_ID_FIELD_DROP'] = df.index.tolist()
flds = df.columns.tolist()
flds.pop(flds.index(gname))
flds_lower = [f.lower() for f in flds]
for f in ['objectid', 'oid', 'fid']:
if f in flds_lower:
idx = flds_lower.index(f)
flds.pop(idx)
flds_lower.pop(idx)
del idx
del f
array = [tuple(row) for row in df[flds].as_matrix()]
geoms = df.geometry.as_arcpy.tolist()
dtypes = []
for idx, a in enumerate(array[0]):
if isinstance(a,
STRING_TYPES):
dtypes.append((flds[idx], '<U%s' % df[flds[idx]].map(len).max()))
elif flds[idx].lower() in ['fid', 'oid', 'objectid']:
dtypes.append((flds[idx], np.int32))
elif isinstance(a,
(int, np.int32)):
dtypes.append((flds[idx], np.int64))
elif isinstance(a,
(float, np.float, np.float64)):
dtypes.append((flds[idx], np.float64))
elif isinstance(a,
(datetime.datetime, pd.datetime)):
dtypes.append((flds[idx], '<M8[us]'))
else:
dtypes.append((flds[idx], type(a)))
del idx, a
array = np.array(array, dtype=dtypes)
del dtypes, flds, flds_lower
with da.InsertCursor(fc, ['SHAPE@']) as icur:
for g in geoms:
if skip_invalid:
try:
icur.insertRow([g])
except: pass
else:
icur.insertRow([g])
desc = arcpy.Describe(fc)
oidField = desc.oidFieldName
del desc
da.ExtendTable(in_table=fc, table_match_field=oidField,
in_array=array, array_match_field='JOIN_ID_FIELD_DROP',
append_only=False)
del df['JOIN_ID_FIELD_DROP']
return fc
def main(*argv):
""" main driver of program """
try:
tds_features = argv[0]
in_fields = str(argv[1]).upper() #'zi001_vsn'.upper()
comparative_features = argv[2]
polygon_grid = argv[3]
output_gdb = argv[4]
distanceTolerance = float(argv[5]) #25 #assumes meters
sample_size = int(
argv[6]
) #2 #if 100 or less, is a percent. if 101 or greater, is a number of features
# Local Variables
#
#output_features = output_grid + "_lines"
sr = arcpy.SpatialReference(3857)
is_percentage = False
results = []
# Logic
#
import datetime
start = datetime.datetime.now()
master_times = datetime.datetime.now()
arcpy.AddMessage("Copying Features.")
tds_features = arcpy.CopyFeatures_management(
tds_features, os.path.join(output_gdb,
"PositionalOffset_Roads"))[0]
polygon_grid = arcpy.CopyFeatures_management(
polygon_grid, os.path.join(output_gdb, "PositionalOffset_Grid"))[0]
arcpy.AddMessage("Total Time to Copy %s" %
(datetime.datetime.now() - master_times))
desc = arcpy.Describe(tds_features)
efields = ['SHAPE', 'OID', 'FID', 'OBJECTID', 'SHAPE_LENGTH']
efields = efields + in_fields.upper().split(';')
if hasattr(desc, 'OIDFieldName'):
efields.append(desc.OIDFieldName)
if hasattr(desc, 'lengthFieldName'):
efields.append(desc.lengthFieldName)
if hasattr(desc, 'areaFieldName'):
efields.append(desc.areaFieldName)
for f in [field.name for field in arcpy.ListFields(tds_features)]:
if f.upper() not in efields:
try:
arcpy.DeleteField_management(tds_features, f)
arcpy.AddMessage(" Deleted " + f)
except:
arcpy.AddMessage("Failed to Delete " + f)
del f
del desc, efields
if sample_size <= 100:
sample_size = sample_size / 100
is_percentage = True
tds_sdf = SpatialDataFrame.from_featureclass(tds_features, sr=sr)
if is_percentage:
sample_size = int(len(tds_sdf) * sample_size)
sampled_sdf = tds_sdf.sample(n=sample_size).copy()
cols = {c: c.upper() for c in sampled_sdf.columns}
sampled_sdf.rename(columns=cols, inplace=True)
# Reproject to SR specified above.
#
sampled_sdf.geometry = sampled_sdf.geometry.projectAs(sr)
comparison_sdf = SpatialDataFrame.from_featureclass(
comparative_features, sr=sr)
cols = {c: c.upper() for c in comparison_sdf.columns}
comparison_sdf.rename(columns=cols, inplace=True)
#comparison_sdf.geometry = comparison_sdf.geometry.projectAs(sr)
grid_sdf = SpatialDataFrame.from_featureclass(polygon_grid, sr=sr)
cols = {c: c.upper() for c in grid_sdf.columns}
grid_sdf.rename(columns=cols, inplace=True)
#grid_sdf.geometry = grid_sdf.geometry.projectAs(sr)
grid_sdf['MEAN_CE90'] = 0.0
grid_sdf['OFFSET_METERS'] = 0.0
grid_sdf['OSM_MATCH'] = ""
compare_sindex = comparison_sdf.sindex
sample_sindex = sampled_sdf.sindex
counter = 0
for idx, row in grid_sdf.iterrows():
print('Processing Grid Number ' + str(counter))
arcpy.AddMessage('Processing Grid Number ' + str(counter))
counter = counter + 1
g = row['SHAPE']
oid = row['OBJECTID']
ext = [g.extent.XMin, g.extent.YMin, g.extent.XMax, g.extent.YMax]
# extract select_compare_df
select_compare_df = comparison_sdf.loc[compare_sindex.intersect(
ext)] # OSM Source
q = select_compare_df.geometry.disjoint(g) == False
select_compare_df = select_compare_df[q].copy()
# extract sample df rows
select_sample_df = sampled_sdf.loc[sample_sindex.intersect(
ext)] # source NGA
q = select_sample_df.geometry.disjoint(g) == False
select_sample_df = select_sample_df[q].copy()
# Find distances to each other.
for idx_s, row_s in select_sample_df.iterrows():
geom = row_s['SHAPE']
sample_oid = row_s['OBJECTID']
print('Processing Feature ' + str(sample_oid))
arcpy.AddMessage('Processing Feature ' + str(sample_oid))
#geom = geom.as_arcpy
first_point = arcpy.PointGeometry(row_s['SHAPE'].firstPoint,
sr)
last_point = arcpy.PointGeometry(row_s['SHAPE'].lastPoint, sr)
source_angle = (first_point.angleAndDistanceTo(last_point)[0] +
360) % 360
buffer_shape = geom.buffer(distanceTolerance)
q = select_compare_df.geometry.disjoint(buffer_shape) == False
for idx_c, row_c in select_compare_df[q].iterrows():
geom_clipped = row_c['SHAPE'].intersect(
geom.buffer(distanceTolerance), 2)
if geom_clipped and geom_clipped.length > 0:
first_point = arcpy.PointGeometry(
geom_clipped.firstPoint, sr)
last_point = arcpy.PointGeometry(
geom_clipped.lastPoint, sr)
comparative_angle = (
first_point.angleAndDistanceTo(last_point)[0] +
360) % 360
if ((comparative_angle <= source_angle + 7.5
and comparative_angle >= source_angle - 7.5) or
(comparative_angle <= source_angle + 180 + 7.5
and comparative_angle >= source_angle + 180 - 7.5)
or
(comparative_angle <= source_angle - 180 + 7.5 and
comparative_angle >= source_angle - 180 - 7.5)):
near_distance = geom.distanceTo(geom_clipped)
#if offset_meters < 0 or near_distance < offset_meters:
#offset_meters = near_distance
results.append(
(sample_oid, row_c['OBJECTID'], oid, "YES",
near_distance)
) # OID of Source (NGA), OID of Grid, Yes, distance to compare feature
del idx_c, row_c
del idx_s
del row_s
del geom
del q
del buffer_shape
del ext
del idx, row
print(datetime.datetime.now() - start)
dtypes = np.dtype([
('SAMPLE_OID', np.int), # SAMPLE OID
('COMPARE_OID', np.int), # COMPARE OID
('GRID_OID', np.int), # GRID OID
('NEAROSM', '|S255'), #
('FEATURE_DISTANCE', np.float64)
])
array = np.array(results, dtype=dtypes)
df = pd.DataFrame(data=array,
columns=[
'SAMPLE_OID', 'COMPARE_OID', 'GRID_OID',
'NEAROSM', 'FEATURE_DISTANCE'
])
del df['NEAROSM']
print("join average grid distance to grid")
sample_mean_distance = df.groupby(
by='GRID_OID', as_index=False)['FEATURE_DISTANCE'].mean().copy()
sample_mean_distance['MEAN_CE90'] = sample_mean_distance[
'FEATURE_DISTANCE']
sample_mean_distance = sample_mean_distance.to_records(index=False)
dt1 = np.dtype([('GRID_OID', '<i4'), ('MEAN_CE90', '<f8')])
join_sample = np.array(sample_mean_distance, dtype=dt1)
oidName = arcpy.Describe(polygon_grid).oidFieldName
da.ExtendTable(in_table=polygon_grid,
table_match_field=oidName,
in_array=join_sample,
array_match_field="GRID_OID",
append_only=False)
del sample_mean_distance
del oidName
del join_sample
del dt1
print("join closest distance to tds_features")
dt2 = np.dtype([('SAMPLE_OID', '<i4'), ('FEATURE_DISTANCE', '<f8')])
tds_join_data = df.groupby(
by='SAMPLE_OID', as_index=False)['FEATURE_DISTANCE'].min().copy()
tds_join_data = tds_join_data.to_records(index=False)
join_sample = np.array(tds_join_data, dtype=dt2)
oidName = arcpy.Describe(tds_features).oidFieldName
da.ExtendTable(in_table=tds_features,
table_match_field=oidName,
in_array=join_sample,
array_match_field="SAMPLE_OID",
append_only=False)
del dt2
del tds_join_data
del join_sample
del oidName
print('return results')
arcpy.SetParameterAsText(7, polygon_grid)
arcpy.SetParameterAsText(8, tds_features)
except arcpy.ExecuteError:
line, filename, synerror = trace()
arcpy.AddError("error on line: %s" % line)
arcpy.AddError("error in file name: %s" % filename)
arcpy.AddError("with error message: %s" % synerror)
arcpy.AddError("ArcPy Error Message: %s" % arcpy.GetMessages(2))
except FunctionError as f_e:
messages = f_e.args[0]
arcpy.AddError("error in function: %s" % messages["function"])
arcpy.AddError("error on line: %s" % messages["line"])
arcpy.AddError("error in file name: %s" % messages["filename"])
arcpy.AddError("with error message: %s" % messages["synerror"])
arcpy.AddError("ArcPy Error Message: %s" % messages["arc"])
except:
line, filename, synerror = trace()
arcpy.AddError("error on line: %s" % line)
arcpy.AddError("error in file name: %s" % filename)
arcpy.AddError("with error message: %s" % synerror)
