def replace_null_values(fc, fields="*", oid_field=None, null_value=0, where_clause=None): """updates a set of rows in chunks """ if fields is None or \ isinstance(fields, list) == False or \ fields == "*": fields = [field.name for field in arcpy.ListFields(fc) \ if field.type not in ('Geometry', 'Blob', 'Raster')] if oid_field is None: oid_field = arcpy.Describe(fc).OIDFieldName chunk_size = calc_chunk_size() if oid_field not in fields: fields.append(oid_field) with da.SearchCursor(fc, fields, where_clause=where_clause) as cursor: search_fields = [ field for field in cursor.fields if field != oid_field ] for group in grouper_it(chunk_size, cursor): df = pd.DataFrame.from_records(group, columns=cursor.fields) for field in search_fields: df.loc[df[field].isnull(), field] = null_value del field array = df.to_records(index=False) da.ExtendTable(fc, oid_field, array, oid_field, False) del array return fc
def extend_table(table, rows=None): """ Adds the required columns to the table and appends new records if given. """ try: if rows is None: rows = [] dtypes = np.dtype([('_ID', np.int), ('MEAN', np.float64), ('MEDIAN', np.float64), ('MODE', np.float64), ('MIN', np.float64), ('MAX', np.float64), ('NO_DATE_CNT', np.int32), ('NO_DATE_PCT', np.float64), ('FEATURE_CNT', np.int32), ('PA_SCORE', np.int32), ("TIER", '|S1024')]) array = np.array(rows, dtypes) da.ExtendTable(table, "OID@", array, "_ID", False) except: line, filename, synerror = trace() raise FunctionError({ "function": "", "line": line, "filename": filename, "synerror": synerror, "arc": str(arcpy.GetMessages(2)) })
def extend_table(rows, table): """ appends the results of the array to the existing table by an objectid """ try: dtypes = np.dtype([ ('_ID', np.int), ('DOM_DATE', '|S48'), ('DOM_DATE_CNT', np.int32), ('DOM_DATE_PER', np.float64), ('DOM_YEAR', np.int32), ('DOM_YEAR_CNT', np.int32), ('DOM_YEAR_PER', np.float64), ('OLDEST_DATE', '|S1024'), ('NEWEST_DATE', '|S1024'), ('NO_DATE_CNT', np.int32), ('NO_DATE_PER', np.float64), ('PCT_2_YEAR', np.float64), ('PCT_5_YEAR', np.float64), ('PCT_10_YEAR', np.float64), ('PCT_15_YEAR', np.float64), ('PCT_15_PLUS_YEAR', np.float64), ('FEATURE_CNT', np.int32), ('CURRENCY_SCORE', np.int32) ]) array = np.array(rows, dtypes) da.ExtendTable(table, "OID@", array, "_ID", False) return table except: line, filename, synerror = trace() raise FunctionError({ "function": "", "line": line, "filename": filename, "synerror": synerror, "arc": str(arcpy.GetMessages(2)) })
def grid_fields(grid): """ Adds fields to a given polygon grid. """ try: dtypes = np.dtype([ ('_ID', np.int), ('DOM_DATE', '|S48'), ('DOM_DATE_CNT', np.int32), ('DOM_DATE_PER', np.float64), ('DOM_YEAR', np.int32), ('DOM_YEAR_CNT', np.int32), ('DOM_YEAR_PER', np.float64), ('OLDEST_DATE', '|S1024'), ('NEWEST_DATE', '|S1024'), ('NO_DATE_CNT', np.int32), ('NO_DATE_PER', np.float64), ('PCT_2_YEAR', np.float64), ('PCT_5_YEAR', np.float64), ('PCT_10_YEAR', np.float64), ('PCT_15_YEAR', np.float64), ('PCT_15_PLUS_YEAR', np.float64), ('FEATURE_CNT', np.int32), ('CURRENCY_SCORE', np.int32) ]) array = np.array([], dtypes) da.ExtendTable(grid, "OID@", array, "_ID", False) return grid except: line, filename, synerror = trace() raise FunctionError({ "function": "grid_fields", "line": line, "filename": filename, "synerror": synerror, "arc": str(arcpy.GetMessages(2)) })
def extend_table(table, rows=None): """ Adds the required columns to the table and appends new records if given. """ try: if rows is None: rows = [] dtypes = np.dtype( [ ('_ID', np.int), ('DOM_SCALE', np.float64), ('DOM_COUNT', np.int32), ('DOM_PER', np.float64), ('MIN_SCALE', np.float64), ('MIN_PER', np.float64), ('MAX_SCALE', np.float64), ('MAX_PER', np.float64), ('CNT_2500', np.int32), ('CNT_5000', np.int32), ('CNT_12500', np.int32), ('CNT_25000', np.int32), ('CNT_50000', np.int32), ('CNT_100000', np.int32), ('CNT_250000', np.int32), ('CNT_500000', np.int32), ('CNT_1000000', np.int32), ('PER_2500', np.float64), ('PER_5000', np.float64), ('PER_12500', np.float64), ('PER_25000', np.float64), ('PER_50000', np.float64), ('PER_100000', np.float64), ('PER_250000', np.float64), ('PER_500000', np.float64), ('PER_1000000', np.float64), ('COUNT', np.int32), ('MISSION_PLANNING', '|S1024'), ('POPULATION_SCALE', '|S1024'), ('THEM_ACC_SCORE', np.float64) ] ) array = np.array(rows, dtypes) da.ExtendTable(table, "OID@", array, "_ID", False) return table except: line, filename, synerror = trace() raise FunctionError( { "function": "extend_table", "line": line, "filename": filename, "synerror": synerror, "arc" : str(arcpy.GetMessages(2)) } )
def insert_new_results(selected_auth_fl, selected_res_fl, authoritative_fc, schema): selected_auth_sdf = SpatialDataFrame.from_featureclass(selected_auth_fl) print(len(selected_auth_sdf)) selected_res_sdf = SpatialDataFrame.from_featureclass(selected_res_fl) print(len(selected_res_sdf)) fields = field_schema.get(schema) #for f in fields: # print(f) # Write this function dtypes = dts.get(schema) fields = field_schema.get(schema) for idx, sel_auth_row in enumerate(selected_auth_sdf.iterrows()): geom = sel_auth_row[1].SHAPE.buffer(-.01) oid = sel_auth_row[1].OBJECTID # print(oid) ext = geom.extent sq = selected_res_sdf['SHAPE'].disjoint(geom) == False df_current = selected_res_sdf[sq].copy() df_current.reset_index(inplace=True) #print(df_current.head()) if len(df_current) > 0: # print("Here") #['MEAN_CE90'] for f in fields: try: cur_val = df_current.loc[0].at[f] #print(cur_val) selected_auth_sdf.at[idx, f] = cur_val except: # break print("Field doesn't exist") insert_df = selected_auth_sdf.drop(['SHAPE'], axis=1, inplace=False) records = insert_df.to_records(index=False) rows = np.array(records, dtype=dtypes) array = rows # np.array(rows, dtypes) da.ExtendTable(authoritative_fc, "OID@", array, "_ID", False) return authoritative_fc
def replace_values(fc, fields="*", oid_field=None, find_value=None, replace_value=0, where_clause=None): """updates a set of rows in chunks """ try: if fields is None or \ isinstance(fields, list) == False or \ fields == "*": fields = [field.name for field in arcpy.ListFields(fc) \ if field.type not in ('Geometry', 'Blob', 'Raster')] if oid_field is None: oid_field = arcpy.Describe(fc).OIDFieldName chunk_size = calc_chunk_size() if oid_field not in fields: fields.append(oid_field) with da.SearchCursor(fc, fields, where_clause=where_clause) as cursor: search_fields = [ field for field in cursor.fields if field != oid_field ] for group in grouper_it(chunk_size, cursor): df = pd.DataFrame.from_records(group, columns=cursor.fields) for field in search_fields: if find_value is None or \ str(find_value).lower() == "none" or \ str(find_value).lower().strip() == "": df.loc[df[field].isnull(), field] = replace_value else: df.loc[df[field] == find_value, field] = replace_value del field array = df.to_records(index=False) da.ExtendTable(fc, oid_field, array, oid_field, False) del array del df return fc except: line, filename, synerror = trace() raise FunctionError({ "function": "replace_values", "line": line, "filename": filename, "synerror": synerror, "arc": str(arcpy.GetMessages(2)) })
def extend_table(fc, array): """ extends to append field and data to an existing table """ try: da.ExtendTable(fc, "OID@", array, "_ID", False) return fc except: line, filename, synerror = trace() raise FunctionError({ "function": "extend_table", "line": line, "filename": filename, "synerror": synerror, "arc": str(arcpy.GetMessages(2)) })
def create_error_fc(outFC, geometryType, sr=None): """ Creates an error feature class for a given geometry type and spatial reference. Output: path to a feature class """ try: if arcpy.Exists(outFC): arcpy.AddMessage("Error Feature Class Already Exists, Recreating...") arcpy.Delete_management(outFC) arcpy.CreateFeatureclass_management(out_path=os.path.dirname(outFC), out_name=os.path.basename(outFC), geometry_type=geometryType.upper(), spatial_reference=sr) narray = np.array([], np.dtype([('_ID', np.int),# _ID will not be included in the extend ('DEFICIENCY', '|S1024'), ('FEATURE_CLASS', '|S254'), ('SUBTYPE', '|S254'), ('ORIG_OID', np.int), ('DEFICIENCY_CNT', np.int) ]) ) da.ExtendTable(outFC, "OID@", narray, "_ID") return outFC except: line, filename, synerror = trace() raise FunctionError( { "function": "", "line": line, "filename": __file__, "synerror": synerror, "arc" : str(arcpy.GetMessages(2)) } )
def extend_table(table, rows=None): """ Adds the required columns to the table and appends new records if given. """ try: if rows is None: rows = [] dtypes = np.dtype([ ('_ID', np.int), ('MEAN_DEF_CNT', np.int32), ('MEDIAN_DEF_CNT', np.float64), ('MIN_DEF_CNT', np.int32), ('MAX_DEF_CNT', np.int32), #STandard deviation ('PRI_NUM_DEF', np.int32), ('SEC_NUM_DEF', np.int32), ('PER_PRI', np.float64), ('PER_SEC', np.float64), ("PRI_ATTR_DEF", '|S20'), # pri_attr ("SEC_ATTR_DEF", '|S20'), ('PRI_ATTR_DEF_PER', np.float64), ('SEC_ATTR_DEF_PER', np.float64), ('FEATURE_CNT', np.int32), ('PRI_ATTR_DEF_CNT', np.float64), ('SEC_ATTR_DEF_CNT', np.float64), ('LC_SCORE', np.int32) ]) array = np.array(rows, dtypes) da.ExtendTable(table, "OID@", array, "_ID", False) return table except: line, filename, synerror = trace() raise FunctionError({ "function": "extend_table", "line": line, "filename": filename, "synerror": synerror, "arc": str(arcpy.GetMessages(2)) })
def extend_table(table, rows=None): """ Adds the required columns to the table and appends new records if given. """ try: if rows is None: rows = [] dtypes = np.dtype([('_ID', np.int), ('TDS_DENSITY', np.float64), ('COMP_DENSITY', np.float64), ('COMPLETENESS_VALUE', np.float64), ('DIFFERENCE', np.float64)]) array = np.array(rows, dtypes) da.ExtendTable(table, "OID@", array, "_ID", False) return table except: line, filename, synerror = trace() raise FunctionError({ "function": "extend_table", "line": line, "filename": filename, "synerror": synerror, "arc": str(arcpy.GetMessages(2)) })
def data_comparison(in_grid, in_fcs, in_old_gdb, in_new_gdb, out_grid, geom_type="POINT", scratchGDB=env.scratchGDB): """ Generates rankings based on a given grid and data type Inputs: in_grid: path to area of interests in_fcs: list of feature class names in_old_gdb: old FGDB path for comparison in_new_gdb: new FGDB path for comparison out_grid: path of the output GDB feature class geom_type: string value of POINT, POLYLINE, or POLYGON """ try: merged_points_n = os.path.join("in_memory", "merged_pts_n") #scratchGDB merged_points_o = os.path.join("in_memory", "merged_pts_o") #scratchGDB temp_out_grid = os.path.join(scratchGDB, "grid") old_stats = os.path.join(scratchGDB, "old_stats") new_stats = os.path.join(scratchGDB, "new_stats") # Copy Grid to Temp Folder temp_out_grid = arcpy.CopyFeatures_management(in_grid, temp_out_grid)[0] # Merge all fcs into one fc merged_points_n, total_n_count = merge_fcs(in_fcs, merged_points_n, gdb=in_new_gdb) merged_points_o, total_o_count = merge_fcs(in_fcs, merged_points_o, gdb=in_old_gdb) # intersect the grid new_pts_int = arcpy.Intersect_analysis( in_features=[merged_points_n, temp_out_grid], out_feature_class=os.path.join(scratchGDB, "new_pts_int"), join_attributes="ONLY_FID")[0] old_pts_int = arcpy.Intersect_analysis( in_features=[merged_points_o, temp_out_grid], out_feature_class=os.path.join(scratchGDB, "old_pts_int"), join_attributes="ONLY_FID")[0] if geom_type.lower() == "point": stat_fields_new = "FID_grid COUNT" stat_fields_old = "FID_grid COUNT" method = ["POINT"] elif geom_type.lower() in ("polyline", 'polygon'): arcpy.AddField_management(old_pts_int, "OLD_LENGTH", "FLOAT") arcpy.CalculateField_management( in_table=old_pts_int, field="OLD_LENGTH", expression="!shape.length@kilometers!", expression_type="PYTHON_9.3", code_block="") arcpy.AddField_management(new_pts_int, "NEW_LENGTH", "FLOAT") arcpy.CalculateField_management( in_table=new_pts_int, field="NEW_LENGTH", expression="!shape.length@kilometers!", expression_type="PYTHON_9.3", code_block="") if geom_type.lower() == "polygon": arcpy.AddField_management(old_pts_int, "OLD_AREA", "FLOAT") #!shape.area@squarekilometers! arcpy.CalculateField_management( in_table=old_pts_int, field="OLD_AREA", expression="!shape.area@squarekilometers!", expression_type="PYTHON_9.3", code_block="") arcpy.AddField_management(new_pts_int, "NEW_AREA", "FLOAT") arcpy.CalculateField_management( in_table=new_pts_int, field="NEW_AREA", expression="!shape.area@squarekilometers!", expression_type="PYTHON_9.3", code_block="") if geom_type.lower() == "polygon": stat_fields_new = "FID_grid COUNT;NEW_LENGTH SUM;NEW_AREA SUM" stat_fields_old = "FID_grid COUNT;OLD_LENGTH SUM;OLD_AREA SUM" method = ['POINT', 'POLYLINE', 'POLYGON'] else: stat_fields_new = "FID_grid COUNT;NEW_LENGTH SUM" stat_fields_old = "FID_grid COUNT;OLD_LENGTH SUM" method = ['POINT', 'POLYLINE'] # get the counts old_stats = arcpy.Statistics_analysis( in_table=old_pts_int, out_table=old_stats, statistics_fields=stat_fields_old, case_field="FID_grid")[0] new_stats = arcpy.Statistics_analysis( in_table=new_pts_int, out_table=new_stats, statistics_fields=stat_fields_new, case_field="FID_grid")[0] # join the old stats to the new stats if geom_type.lower() == "polygon": arcpy.AlterField_management(new_stats, field="SUM_NEW_LENGTH", new_field_name="NEW_LENGTH") arcpy.AlterField_management(new_stats, field="SUM_NEW_AREA", new_field_name="NEW_AREA") out_fields = [ 'FID_grid', 'FREQUENCY', 'SUM_OLD_LENGTH', 'SUM_OLD_AREA' ] ndt = np.dtype([('FID_grid', '<i4'), ('OLD_FREQUENCY', '<i4'), ('OLD_LENGTH', np.float64), ('OLD_AREA', np.float64)]) export_fields = [ 'FID_grid', 'FREQUENCY', 'OLD_FREQUENCY', 'OLD_LENGTH', 'NEW_LENGTH', 'OLD_AREA', 'NEW_AREA', 'SCORE', 'RANKING' ] elif geom_type.lower() == "point": out_fields = ['FID_grid', 'FREQUENCY'] ndt = np.dtype([('FID_grid', '<i4'), ('OLD_FREQUENCY', '<i4')]) export_fields = [ 'FID_grid', 'FREQUENCY', 'OLD_FREQUENCY', 'SCORE', 'RANKING' ] elif geom_type.lower() == "polyline": arcpy.AlterField_management(new_stats, field="SUM_NEW_LENGTH", new_field_name="NEW_LENGTH") out_fields = ['FID_grid', 'FREQUENCY', 'SUM_OLD_LENGTH'] ndt = np.dtype([('FID_grid', '<i4'), ('OLD_FREQUENCY', '<i4'), ('OLD_LENGTH', np.float64)]) export_fields = [ 'FID_grid', 'FREQUENCY', 'OLD_FREQUENCY', 'OLD_LENGTH', 'NEW_LENGTH', 'SCORE', 'RANKING' ] old_array = da.TableToNumPyArray(in_table=old_stats, field_names=out_fields) old_array.dtype = ndt # Add SCORE and RANKING fields and remove unneeded fields da.ExtendTable(new_stats, "FID_grid", old_array, "FID_grid", False) array = np.array([], np.dtype([('_id', np.int32), ('SCORE', np.float64), ('RANKING', np.int64)])) da.ExtendTable(new_stats, arcpy.Describe(new_stats).OIDFieldName, array, "_id", False) arcpy.DeleteField_management(new_stats, ['COUNT_FID_grid']) array = da.TableToNumPyArray(in_table=new_stats, field_names=export_fields, null_value=0) # Calculate the rankings tcsv, column_list = calculate_frequency_ranking(array=array, methods=method) array = da.TableToNumPyArray(tcsv, column_list) da.ExtendTable(temp_out_grid, arcpy.Describe(temp_out_grid).OIDFieldName, array, "FID_grid") # Clean up NULL values if geom_type.lower() == "point": sql = """RANKING IS NULL""" fields = ['RANKING'] elif geom_type.lower() == "polyline": sql = """RANKING IS NULL OR RANKING_LENGTH IS NULL""" fields = ['RANKING', 'RANKING_LENGTH'] elif geom_type.lower() == "polygon": sql = """RANKING IS NULL OR RANKING_LENGTH IS NULL OR RANKING_AREA IS NULL""" fields = ['RANKING', 'RANKING_LENGTH', 'RANKING_AREA'] with da.UpdateCursor(temp_out_grid, fields, where_clause=sql) as urows: for row in urows: if geom_type.lower() == "point": row[0] = 0 elif geom_type.lower() == "polyline": if row[0] is None: row[0] = 0 if row[1] is None: row[1] = 0 elif geom_type.lower() == "polygon": if row[0] is None: row[0] = 0 if row[1] is None: row[1] = 0 if row[2] is None: row[2] = 0 urows.updateRow(row) del row del urows # return the output grid return arcpy.CopyFeatures_management(temp_out_grid, out_grid)[0] except: line, filename, synerror = trace() raise FunctionError({ "function": "data_comparison", "line": line, "filename": filename, "synerror": synerror, "arc": str(arcpy.GetMessages(2)) })
def main(*argv): """ main driver of program """ try: fc = argv[0] find_value = argv[1] #None # fields = str(argv[2]).split(';') #str("name;name_en").split(";") # copy_fc = argv[3] feat_count = arcpy.GetCount_management(fc) index = int(feat_count.getOutput(0)) - 1 text_fields = ",".join(fields) flag_field = "FLAGGED_DATA" oid_field = None scratchGDB = env.scratchGDB #copy_fc = os.path.join(scratchGDB, os.path.basename(fc)) isnumber = False # Logic # if find_value is None or \ find_value == "" or \ str(find_value).lower() == "none": find_value = None elif find_value.startswith("-") and \ find_value.find(".") < -1 and \ find_value[1:].isdigit(): find_value = int(find_value) isnumber = True elif find_value.find(".") < -1 and \ find_value.isdigit(): find_value = int(find_value) isnumber = True if isnumber == False: try: find_value = float(find_value) isnumber = True except: isnumber = False if arcpy.Exists(copy_fc): arcpy.Delete_management(copy_fc) copy_fc = copy_rows( source=fc, destination=copy_fc) # Instead of arcpy.CopyFeatures() #arcpy.CopyFeatures_management(fc, copy_fc) #if check_shape: # fields.append("SHAPE@") if oid_field is None: oid_field = arcpy.Describe(fc).OIDFieldName if oid_field not in fields: fields.append(oid_field) chunk_size = calc_chunk_size() count = 0 dtypes = np.dtype([('_ID', np.int), ('NULL_COLUMNS', '|S255'), ('FLAG_FIELDS', '|S255'), ('NULL_COUNT', np.int32), ('FLAGGED_DATA', np.int32)]) #('SELECTED_FIELDS', '|S1024'), #('FLAGGED_FIELDS', '|S1024'), with da.SearchCursor(copy_fc, fields, where_clause=None) as cursor: search_fields = [ field for field in cursor.fields if field != oid_field ] for group in grouper_it(chunk_size, cursor): flagged_fields = [] count += 1 df = pd.DataFrame.from_records(group, columns=cursor.fields) df['NULL_COLUMNS'] = df.apply( lambda x: ','.join(x[x.isin([find_value])].index), axis=1) #df.apply(lambda x: ','.join(x[x.isnull()].index),axis=1)# df['FLAG_FIELDS'] = text_fields #df['SELECTED_FIELDS'] = text_fields #df['FLAGGED_FIELDS'] = text_fields if find_value is None: df['NULL_COUNT'] = len(df.columns) - df.count(axis=1) elif find_value and isnumber: df['NULL_COUNT'] = (df[fields] == find_value).sum(axis=1) elif find_value and isnumber == False: df['NULL_COUNT'] = (df[fields] == find_value).sum(axis=1) df['NULL_COUNT'] = df[fields].isin([find_value ]).sum(axis=1) df[flag_field] = 0 df.loc[df['NULL_COUNT'] > 0, flag_field] = 1 for fld in search_fields: del df[fld] array = df.to_records(index=False).tolist() array = np.array(array, dtypes) da.ExtendTable(copy_fc, oid_field, array, '_ID', False) del group del df del array #arcpy.SetParameterAsText(3, copy_fc) except arcpy.ExecuteError: line, filename, synerror = trace() arcpy.AddError("error on line: %s" % line) arcpy.AddError("error in file name: %s" % filename) arcpy.AddError("with error message: %s" % synerror) arcpy.AddError("ArcPy Error Message: %s" % arcpy.GetMessages(2)) except FunctionError as f_e: messages = f_e.args[0] arcpy.AddError("error in function: %s" % messages["function"]) arcpy.AddError("error on line: %s" % messages["line"]) arcpy.AddError("error in file name: %s" % messages["filename"]) arcpy.AddError("with error message: %s" % messages["synerror"]) arcpy.AddError("ArcPy Error Message: %s" % messages["arc"]) except: line, filename, synerror = trace() arcpy.AddError("error on line: %s" % line) arcpy.AddError("error in file name: %s" % filename) arcpy.AddError("with error message: %s" % synerror)
def main(*argv): """ main driver of program """ try: old_fc = argv[0] new_fc = argv[1] polygon_grid = argv[2] fields = str(argv[3]).split(';') #argv[3] out_gdb = argv[4] # Local Variables # scratchGDB = env.scratchGDB scratchFolder = env.scratchFolder copy_grid = os.path.join(out_gdb, "grid") copy_old_fc = os.path.join(scratchGDB, os.path.basename(old_fc) + "_old") copy_new_fc = os.path.join(scratchGDB, os.path.basename(new_fc) + "_new") intOld = os.path.join(scratchGDB, "intold") intNew = os.path.join(scratchGDB, "intnew") sumOld = os.path.join(scratchGDB, "sum_old") sumNew = os.path.join(scratchGDB, "sum_new") temp_csv = os.path.join(scratchFolder, "temp_csv.csv") # Logic # # Validate that fields exist in both tables old_fc_flds = { field.name for field in arcpy.ListFields(old_fc) if field.name in fields } new_fc_flds = { field.name for field in arcpy.ListFields(new_fc) if field.name in fields } if len(new_fc_flds) != len(old_fc_flds): missing = list(old_fc_flds - new_fc_flds) + list(new_fc_flds - old_fc_flds) arcpy.AddWarning("Comparison Fields Missing: %s" % ",".join(missing)) fields = [fields.remove(v) for v in missing if v in fields] if len(fields) == 0: raise Exception( "All fields provided do not exist in each dataset. Nothing to compare." ) #copy the data copy_grid = arcpy.CopyFeatures_management(polygon_grid, copy_grid)[0] copy_old_fc = arcpy.FeatureClassToFeatureClass_conversion( old_fc, out_path=os.path.dirname(copy_old_fc), out_name=os.path.basename(copy_old_fc))[0] copy_new_fc = arcpy.FeatureClassToFeatureClass_conversion( new_fc, out_path=os.path.dirname(copy_new_fc), out_name=os.path.basename(copy_new_fc))[0] # get the null counts oldResult = calculate_nulls(copy_old_fc, fields) newResult = calculate_nulls(copy_new_fc, fields) # Intersect Polygon with old/new intOld = arcpy.Intersect_analysis(in_features=[copy_grid, copy_old_fc], out_feature_class=intOld, join_attributes="ALL", cluster_tolerance="-1 Unknown", output_type="INPUT")[0] intNew = arcpy.Intersect_analysis(in_features=[copy_grid, copy_new_fc], out_feature_class=intNew, join_attributes="ALL", cluster_tolerance="-1 Unknown", output_type="INPUT")[0] # Aggregavte and Average out ranking by grid OID case_field = "FID_%s" % os.path.basename(copy_grid) sumOld = arcpy.Statistics_analysis( in_table=intOld, out_table=sumOld, statistics_fields="NULL_COUNT SUM;PERCENT_COMP MEAN;RANKING MEAN", case_field=case_field)[0] sumNew = arcpy.Statistics_analysis( in_table=intNew, out_table=sumNew, statistics_fields="NULL_COUNT SUM;PERCENT_COMP MEAN;RANKING MEAN", case_field=case_field)[0] array_old = da.TableToNumPyArray(sumOld, [ case_field, 'SUM_NULL_COUNT', 'MEAN_PERCENT_COMP', 'MEAN_RANKING' ]) dtype_old = [('FID_grid', '<i4'), ('NULL_COUNT_OLD', '<f8'), ('PERCENT_COMP_OLD', '<f8'), ('RANKING_OLD', '<f8')] array_old.dtype = dtype_old array_new = da.TableToNumPyArray(sumNew, [ case_field, 'SUM_NULL_COUNT', 'MEAN_PERCENT_COMP', 'MEAN_RANKING' ]) dtype_new = [('FID_grid', '<i4'), ('NULL_COUNT_NEW', '<f8'), ('PERCENT_COMP_NEW', '<f8'), ('RANKING_NEW', '<f8')] array_new.dtype = dtype_new # Join stats table back to grid df_old = pd.DataFrame(data=array_old, columns=array_old.dtype.fields.keys()) df_new = pd.DataFrame(data=array_new, columns=array_new.dtype.fields.keys()) join_df = df_new.set_index('FID_grid').join( df_old.set_index('FID_grid')) join_df['FID_grid'] = join_df.index dtype_join = list(set(dtype_new + dtype_old)) join_df.loc[(join_df['RANKING_OLD'].isnull()), 'RANKING_OLD'] = 0 join_df.loc[(join_df['RANKING_NEW'].isnull()), 'RANKING_NEW'] = 0 join_df[ 'DIFF_RANKING'] = join_df['RANKING_NEW'] - join_df['RANKING_OLD'] if os.path.isfile(temp_csv): os.remove(temp_csv) join_df.to_csv( temp_csv, columns=[case_field, 'RANKING_OLD', 'RANKING_NEW', 'DIFF_RANKING']) array = da.TableToNumPyArray( temp_csv, [case_field, 'RANKING_OLD', 'RANKING_NEW', 'DIFF_RANKING']) oid = arcpy.Describe(copy_grid).OIDFieldName da.ExtendTable(copy_grid, oid, array, case_field, False) with arcpy.da.UpdateCursor( copy_grid, ['DIFF_RANKING', 'RANKING_OLD', 'RANKING_NEW'], where_clause= "DIFF_RANKING IS NULL or RANKING_OLD is NULL or RANKING_NEW is NULL" ) as urows: for urow in urows: if urow[0] is None: urow[0] = 0 if urow[1] is None: urow[1] = 0 if urow[2] is None: urow[2] = 0 #urow[0] = 0 urows.updateRow(urow) del urow del urows if os.path.isfile(temp_csv): os.remove(temp_csv) # Output # arcpy.SetParameterAsText(5, copy_grid) except arcpy.ExecuteError: line, filename, synerror = trace() arcpy.AddError("error on line: %s" % line) arcpy.AddError("error in file name: %s" % filename) arcpy.AddError("with error message: %s" % synerror) arcpy.AddError("ArcPy Error Message: %s" % arcpy.GetMessages(2)) except FunctionError as f_e: messages = f_e.args[0] arcpy.AddError("error in function: %s" % messages["function"]) arcpy.AddError("error on line: %s" % messages["line"]) arcpy.AddError("error in file name: %s" % messages["filename"]) arcpy.AddError("with error message: %s" % messages["synerror"]) arcpy.AddError("ArcPy Error Message: %s" % messages["arc"]) except: line, filename, synerror = trace() arcpy.AddError("error on line: %s" % line) arcpy.AddError("error in file name: %s" % filename) arcpy.AddError("with error message: %s" % synerror)
def _arcpy_to_featureclass(df, out_name, out_location=None, overwrite=True, out_sr=None, skip_invalid=True): """ """ import arcgis import numpy as np import datetime from arcpy import da from arcgis.features import SpatialDataFrame gtype = df.geometry_type.upper() gname = df.geometry.name df = df.copy() if overwrite and \ arcpy.Exists(os.path.join(out_location, out_name)): arcpy.Delete_management(os.path.join(out_location, out_name)) elif overwrite == False and \ arcpy.Exists(os.path.join(out_location, out_name)): raise Exception("Dataset exists, please provide a new out_name or location.") if out_sr is None: try: if isinstance(df.sr, dict): sr = arcgis.geometry.SpatialReference(df.sr).as_arcpy elif isinstance(df.sr, arcgis.geometry.SpatialReference): sr = df.sr.as_arcpy except: sr = arcpy.SpatialReference(4326) else: if isinstance(df.sr, dict): sr = arcgis.geometry.SpatialReference(df.sr).as_arcpy elif isinstance(df.sr, arcgis.geometry.SpatialReference): sr = df.sr.as_arcpy elif isinstance(out_sr, arcpy.SpatialReference): sr = out_sr fc = arcpy.CreateFeatureclass_management(out_path=out_location, out_name=out_name, geometry_type=gtype.upper(), spatial_reference=sr)[0] df['JOIN_ID_FIELD_DROP'] = df.index.tolist() flds = df.columns.tolist() flds.pop(flds.index(gname)) flds_lower = [f.lower() for f in flds] for f in ['objectid', 'oid', 'fid']: if f in flds_lower: idx = flds_lower.index(f) flds.pop(idx) flds_lower.pop(idx) del idx del f array = [tuple(row) for row in df[flds].as_matrix()] geoms = df.geometry.as_arcpy.tolist() dtypes = [] for idx, a in enumerate(array[0]): if isinstance(a, STRING_TYPES): dtypes.append((flds[idx], '<U%s' % df[flds[idx]].map(len).max())) elif flds[idx].lower() in ['fid', 'oid', 'objectid']: dtypes.append((flds[idx], np.int32)) elif isinstance(a, (int, np.int32)): dtypes.append((flds[idx], np.int64)) elif isinstance(a, (float, np.float, np.float64)): dtypes.append((flds[idx], np.float64)) elif isinstance(a, (datetime.datetime, pd.datetime)): dtypes.append((flds[idx], '<M8[us]')) else: dtypes.append((flds[idx], type(a))) del idx, a array = np.array(array, dtype=dtypes) del dtypes, flds, flds_lower with da.InsertCursor(fc, ['SHAPE@']) as icur: for g in geoms: if skip_invalid: try: icur.insertRow([g]) except: pass else: icur.insertRow([g]) desc = arcpy.Describe(fc) oidField = desc.oidFieldName del desc da.ExtendTable(in_table=fc, table_match_field=oidField, in_array=array, array_match_field='JOIN_ID_FIELD_DROP', append_only=False) del df['JOIN_ID_FIELD_DROP'] return fc
def main(*argv): """ main driver of program """ try: tds_features = argv[0] in_fields = str(argv[1]).upper() #'zi001_vsn'.upper() comparative_features = argv[2] polygon_grid = argv[3] output_gdb = argv[4] distanceTolerance = float(argv[5]) #25 #assumes meters sample_size = int( argv[6] ) #2 #if 100 or less, is a percent. if 101 or greater, is a number of features # Local Variables # #output_features = output_grid + "_lines" sr = arcpy.SpatialReference(3857) is_percentage = False results = [] # Logic # import datetime start = datetime.datetime.now() master_times = datetime.datetime.now() arcpy.AddMessage("Copying Features.") tds_features = arcpy.CopyFeatures_management( tds_features, os.path.join(output_gdb, "PositionalOffset_Roads"))[0] polygon_grid = arcpy.CopyFeatures_management( polygon_grid, os.path.join(output_gdb, "PositionalOffset_Grid"))[0] arcpy.AddMessage("Total Time to Copy %s" % (datetime.datetime.now() - master_times)) desc = arcpy.Describe(tds_features) efields = ['SHAPE', 'OID', 'FID', 'OBJECTID', 'SHAPE_LENGTH'] efields = efields + in_fields.upper().split(';') if hasattr(desc, 'OIDFieldName'): efields.append(desc.OIDFieldName) if hasattr(desc, 'lengthFieldName'): efields.append(desc.lengthFieldName) if hasattr(desc, 'areaFieldName'): efields.append(desc.areaFieldName) for f in [field.name for field in arcpy.ListFields(tds_features)]: if f.upper() not in efields: try: arcpy.DeleteField_management(tds_features, f) arcpy.AddMessage(" Deleted " + f) except: arcpy.AddMessage("Failed to Delete " + f) del f del desc, efields if sample_size <= 100: sample_size = sample_size / 100 is_percentage = True tds_sdf = SpatialDataFrame.from_featureclass(tds_features, sr=sr) if is_percentage: sample_size = int(len(tds_sdf) * sample_size) sampled_sdf = tds_sdf.sample(n=sample_size).copy() cols = {c: c.upper() for c in sampled_sdf.columns} sampled_sdf.rename(columns=cols, inplace=True) # Reproject to SR specified above. # sampled_sdf.geometry = sampled_sdf.geometry.projectAs(sr) comparison_sdf = SpatialDataFrame.from_featureclass( comparative_features, sr=sr) cols = {c: c.upper() for c in comparison_sdf.columns} comparison_sdf.rename(columns=cols, inplace=True) #comparison_sdf.geometry = comparison_sdf.geometry.projectAs(sr) grid_sdf = SpatialDataFrame.from_featureclass(polygon_grid, sr=sr) cols = {c: c.upper() for c in grid_sdf.columns} grid_sdf.rename(columns=cols, inplace=True) #grid_sdf.geometry = grid_sdf.geometry.projectAs(sr) grid_sdf['MEAN_CE90'] = 0.0 grid_sdf['OFFSET_METERS'] = 0.0 grid_sdf['OSM_MATCH'] = "" compare_sindex = comparison_sdf.sindex sample_sindex = sampled_sdf.sindex counter = 0 for idx, row in grid_sdf.iterrows(): print('Processing Grid Number ' + str(counter)) arcpy.AddMessage('Processing Grid Number ' + str(counter)) counter = counter + 1 g = row['SHAPE'] oid = row['OBJECTID'] ext = [g.extent.XMin, g.extent.YMin, g.extent.XMax, g.extent.YMax] # extract select_compare_df select_compare_df = comparison_sdf.loc[compare_sindex.intersect( ext)] # OSM Source q = select_compare_df.geometry.disjoint(g) == False select_compare_df = select_compare_df[q].copy() # extract sample df rows select_sample_df = sampled_sdf.loc[sample_sindex.intersect( ext)] # source NGA q = select_sample_df.geometry.disjoint(g) == False select_sample_df = select_sample_df[q].copy() # Find distances to each other. for idx_s, row_s in select_sample_df.iterrows(): geom = row_s['SHAPE'] sample_oid = row_s['OBJECTID'] print('Processing Feature ' + str(sample_oid)) arcpy.AddMessage('Processing Feature ' + str(sample_oid)) #geom = geom.as_arcpy first_point = arcpy.PointGeometry(row_s['SHAPE'].firstPoint, sr) last_point = arcpy.PointGeometry(row_s['SHAPE'].lastPoint, sr) source_angle = (first_point.angleAndDistanceTo(last_point)[0] + 360) % 360 buffer_shape = geom.buffer(distanceTolerance) q = select_compare_df.geometry.disjoint(buffer_shape) == False for idx_c, row_c in select_compare_df[q].iterrows(): geom_clipped = row_c['SHAPE'].intersect( geom.buffer(distanceTolerance), 2) if geom_clipped and geom_clipped.length > 0: first_point = arcpy.PointGeometry( geom_clipped.firstPoint, sr) last_point = arcpy.PointGeometry( geom_clipped.lastPoint, sr) comparative_angle = ( first_point.angleAndDistanceTo(last_point)[0] + 360) % 360 if ((comparative_angle <= source_angle + 7.5 and comparative_angle >= source_angle - 7.5) or (comparative_angle <= source_angle + 180 + 7.5 and comparative_angle >= source_angle + 180 - 7.5) or (comparative_angle <= source_angle - 180 + 7.5 and comparative_angle >= source_angle - 180 - 7.5)): near_distance = geom.distanceTo(geom_clipped) #if offset_meters < 0 or near_distance < offset_meters: #offset_meters = near_distance results.append( (sample_oid, row_c['OBJECTID'], oid, "YES", near_distance) ) # OID of Source (NGA), OID of Grid, Yes, distance to compare feature del idx_c, row_c del idx_s del row_s del geom del q del buffer_shape del ext del idx, row print(datetime.datetime.now() - start) dtypes = np.dtype([ ('SAMPLE_OID', np.int), # SAMPLE OID ('COMPARE_OID', np.int), # COMPARE OID ('GRID_OID', np.int), # GRID OID ('NEAROSM', '|S255'), # ('FEATURE_DISTANCE', np.float64) ]) array = np.array(results, dtype=dtypes) df = pd.DataFrame(data=array, columns=[ 'SAMPLE_OID', 'COMPARE_OID', 'GRID_OID', 'NEAROSM', 'FEATURE_DISTANCE' ]) del df['NEAROSM'] print("join average grid distance to grid") sample_mean_distance = df.groupby( by='GRID_OID', as_index=False)['FEATURE_DISTANCE'].mean().copy() sample_mean_distance['MEAN_CE90'] = sample_mean_distance[ 'FEATURE_DISTANCE'] sample_mean_distance = sample_mean_distance.to_records(index=False) dt1 = np.dtype([('GRID_OID', '<i4'), ('MEAN_CE90', '<f8')]) join_sample = np.array(sample_mean_distance, dtype=dt1) oidName = arcpy.Describe(polygon_grid).oidFieldName da.ExtendTable(in_table=polygon_grid, table_match_field=oidName, in_array=join_sample, array_match_field="GRID_OID", append_only=False) del sample_mean_distance del oidName del join_sample del dt1 print("join closest distance to tds_features") dt2 = np.dtype([('SAMPLE_OID', '<i4'), ('FEATURE_DISTANCE', '<f8')]) tds_join_data = df.groupby( by='SAMPLE_OID', as_index=False)['FEATURE_DISTANCE'].min().copy() tds_join_data = tds_join_data.to_records(index=False) join_sample = np.array(tds_join_data, dtype=dt2) oidName = arcpy.Describe(tds_features).oidFieldName da.ExtendTable(in_table=tds_features, table_match_field=oidName, in_array=join_sample, array_match_field="SAMPLE_OID", append_only=False) del dt2 del tds_join_data del join_sample del oidName print('return results') arcpy.SetParameterAsText(7, polygon_grid) arcpy.SetParameterAsText(8, tds_features) except arcpy.ExecuteError: line, filename, synerror = trace() arcpy.AddError("error on line: %s" % line) arcpy.AddError("error in file name: %s" % filename) arcpy.AddError("with error message: %s" % synerror) arcpy.AddError("ArcPy Error Message: %s" % arcpy.GetMessages(2)) except FunctionError as f_e: messages = f_e.args[0] arcpy.AddError("error in function: %s" % messages["function"]) arcpy.AddError("error on line: %s" % messages["line"]) arcpy.AddError("error in file name: %s" % messages["filename"]) arcpy.AddError("with error message: %s" % messages["synerror"]) arcpy.AddError("ArcPy Error Message: %s" % messages["arc"]) except: line, filename, synerror = trace() arcpy.AddError("error on line: %s" % line) arcpy.AddError("error in file name: %s" % filename) arcpy.AddError("with error message: %s" % synerror)