def lineErrorsExportToQCGDB():
rev_table_main = os.path.join(reviewerSessionGDB, "REVTABLEMAIN")
rev_table_line = os.path.join(reviewerSessionGDB, "REVTABLELINE")
print("Exporting the data reviewer lines.")
if (Exists(multipart_line_errors)):
try:
Delete_management(multipart_line_errors)
except:
print("The feature class at: " + str(multipart_line_errors) +
" already exists and could not be deleted.")
else:
pass
if (Exists(single_part_line_errors)):
try:
Delete_management(single_part_line_errors)
except:
print("The feature class at: " + str(single_part_line_errors) +
" already exists and could not be deleted.")
else:
pass
Copy_management(rev_table_line, multipart_line_errors)
JoinField_management(multipart_line_errors, rev_join_field1,
rev_table_main, rev_join_field2)
MultipartToSinglepart_management(multipart_line_errors,
single_part_line_errors)
def FindOverlapsSausages():
'''KDOT Checks are already simplifying lines
technically this is an unnecessary step
and it is a time consuming, resource intensive process
KDOT will run this script on lines that have already been simplified
Data stewards are encouraged to simplify lines to improve their data quality and system performance
using a tool or the following command, which is commented out'''
#SimplifyLine_cartography(inroadcenterlines, ", algorithm="POINT_REMOVE", tolerance="3 Feet", error_resolving_option="RESOLVE_ERRORS", collapsed_point_option="NO_KEEP", error_checking_option="CHECK")
#Check to Make sure that Shape_Length is the appropriate geometry column length in all geodatabases
MakeFeatureLayer_management(roadcenterlines, "RoadCenterlinesS3_L10", "Shape_Length>=10")
#test overlapping with a centerline midpoint
FeatureVerticesToPoints_management("RoadCenterlinesS3_L10", "in_memory/RoadCenterlinesS3_L10_Mid", point_location="MID")
#Get the count of roads within a distance of the centerline midpoint
SpatialJoin_analysis("RoadCenterlinesS3_L10_Mid", roadcenterlines, "in_memory/ValidateOverlaps2", "JOIN_ONE_TO_ONE", "KEEP_ALL", '#', "INTERSECT", "2 Feet", "Distance")
#return the midpoints with a count greater than 1 - indicating possible overlaps
#output this next line into validation geodatabase and add to arcmap for user interface
FeatureClassToFeatureClass_conversion("in_memory/ValidateOverlaps2", r"C:\temp\New File Geodatabase (2).gdb", "ValidateOverlaps",""""Join_Count" > 1""")
#may need to dev code to test for endpoints
#clean up in memory artifact
Delete_management("ValidateOverlaps2")
#In aggregated data, county boundary overlaps are expected. It's also assumed that county bound left right should be coded correctly.
#the following feature layer are the most important overlaps for review
MakeFeatureLayer_management("ValidateOverlaps", "ValidateOverlaps_Non_County_Boundary", "COUNTY_L = COUNTY_R")
#now using same midpoints look for sausages, midpoints less than 60 feet apart, might explore different distances under 60 feet
#it might help and also hurt to lengthen the minimum segment length to longer than 10 feet for this, test a little more
SpatialJoin_analysis("RoadCenterlinesS3_L10_Mid", roadcenterlines, "in_memory/ValidateSausages60", "JOIN_ONE_TO_ONE", "KEEP_ALL", '#', "INTERSECT", "120 Feet", "Distance")
FeatureClassToFeatureClass_conversion("in_memory/ValidateSausages60", r"C:\temp\New File Geodatabase (2).gdb", "ValidateSausages_60ft",""""Join_Count" > 1""")
Delete_management("ValidateSausages60")
MakeFeatureLayer_management("ValidateSausages_60ft", "ValidateSausages_Non_County_Boundary", "COUNTY_L = COUNTY_R")
def UniqueIDgen():
for i in range(87, 88):
c = str(i).zfill(3)
#print "filling in unique Route IDs for county %s" %c
expression = "LRS_ROUTE_PREFIX = 'L' AND LRS_COUNTY_PRE = '%s'" % c
layer = "County" + c
MakeFeatureLayer_management(target, layer, expression)
#this part of the script performs a couple types of dissolves to create a unique set of numbers in 4 characters for every route in a county
#first, do an unsplit dissolve for each local road in the county based on the RD, STS, and POD fields
#this creates nice segments from which to build the destination routes
Dissolve_management(layer, "in_memory/" + layer + "d1",
"RD;STS;POD;LRS_COUNTY_PRE", "GCID COUNT",
"SINGLE_PART", "UNSPLIT_LINES")
#add, calculate, and index a field for a join operation
#cant add index to in memory database so skip that part
AddField_management(layer + "d1", "ConCatRtName", "TEXT", "", "", "50",
"", "NULLABLE", "NON_REQUIRED", "")
AddField_management(layer + "d1", "RouteNum1", "TEXT", "", "", "6", "",
"NULLABLE", "NON_REQUIRED", "")
AddField_management(layer + "d1", "UniqueNum1", "TEXT", "", "", "3",
"", "NULLABLE", "NON_REQUIRED", "")
CalculateField_management(
layer + "d1", "ConCatRtName",
"""[LRS_COUNTY_PRE]&[RD] & [STS] & [POD] """, "VB", "")
#dissolve the unsplit dissolve layer to a multipart, full dissolve to get unique road names
Dissolve_management(layer + "d1", "in_memory/" + layer + "d2",
"ConCatRtName;RouteNum1", "", "MULTI_PART",
"DISSOLVE_LINES")
#A spatial sort here might be nice
#Calculate a unique 4 digit number for each road name
#I'm just using the Object ID to calculate the unique number string, with a spatial sort another incrementation method would be needed
#the road names should mostly be unique, so a spatial sort at this level would only be beneficial of there is POD field is the only road name distinction
#otherwise an attribute sort would be sufficient, if necessary
CalculateField_management("in_memory/" + layer + "d2", "RouteNum1",
"str(!OBJECTID!).zfill(4)", "PYTHON_9.3", "")
# add the unique id field and increment each duplicate road name part
# calculate that unique number back to the split dissolve
AddJoin_management(layer + "d1", "ConCatRtName", "County087d2",
"ConCatRtName", "KEEP_ALL")
CalculateField_management(layer + "d1", layer + "d1.RouteNum1",
"[" + layer + "d2.RouteNum1]", "VB", "")
#AddField_management("in_memory/"+layer+"d2", "UniqueNum1", "TEXT", "", "", "3", "", "NULLABLE", "NON_REQUIRED", "")
RemoveJoin_management(layer + "d1")
#try this spatial sort thing here
Sort_management("in_memory/" + layer + "d1",
"in_memory/" + layer + "d1_Sort",
"Shape ASCENDING;RouteNum1 ASCENDING", "LL")
#now we run the incrementer to calcualte the unique ID's
#the incrementer isnt working here, but it is calculating a unique ID on for the segments, and it is going it better and much faster than the join method
#it might be better to use the incrementer to do this calculation on the sorted table, then figure out the unique ID
Incrementer("in_memory/" + layer + "d1")
Delete_management("in_memory/" + layer + "d1")
Delete_management("in_memory/" + layer + "d2")
Delete_management("in_memory/" + layer + "d2_Sort")
def make_vis(mxd_cur, df, listlyr):
for str_lyr in listlyr:
lyr_cur = mapping.ListLayers(mxd_cur, str_lyr, df)[0]
lyr_cur.visible = True
Delete_management(lyr_cur)
RefreshTOC()
RefreshActiveView()
def preparingSourceCountyData():
print("Starting the preparingSourceCountyData function!")
if Exists(preRouteSourceCRML):
try:
Delete_management(preRouteSourceCRML)
except:
print("Could not delete the features located at: " +
str(preRouteSourceCRML) + ".")
else:
pass
# Make a copy
CopyFeatures_management(routesSourceCountyLRSArnold, preRouteSourceCRML)
# Remove unnecessary fields
preRouteSourceCRMLDescription = Describe(preRouteSourceCRML)
preRouteSourceCRMLOIDFieldName = preRouteSourceCRMLDescription.OIDFieldName
preRouteSourceCRMLShapeFieldName = preRouteSourceCRMLDescription.shapeFieldName
preRouteSourceCRMLShapeAndOIDFieldNames = [
preRouteSourceCRMLOIDFieldName, preRouteSourceCRMLShapeFieldName
]
preRouteSourceCRMLFieldObjectsList = ListFields(preRouteSourceCRML)
preRouteSourceFieldNames = [
x.name for x in preRouteSourceCRMLFieldObjectsList
]
fieldNamesToKeep = [
y for y in preRouteSourceFieldNames if y in preRouteSourceCRMLFields
or y in preRouteSourceCRMLShapeAndOIDFieldNames
]
fieldNamesToRemove = [
z for z in preRouteSourceFieldNames if z not in fieldNamesToKeep
]
for fieldNameItem in fieldNamesToRemove:
DeleteField_management(preRouteSourceCRML, fieldNameItem)
print("Done deleting unnecessary fields.")
MakeFeatureLayer_management(preRouteSourceCRML, fcAsFeatureLayerLG)
selectionQueryL1 = """ SourceRouteId IS NULL OR LRS_ROUTE_PREFIX IN ('I', 'U', 'K') """
SelectLayerByAttribute(fcAsFeatureLayerLG, NEW_SELECTION_CONST,
selectionQueryL1)
CopyFeatures_management(fcAsFeatureLayerLG, stateRoutesAndNullRouteIDs)
DeleteRows_management(fcAsFeatureLayerLG)
selectionQueryL2 = """ SourceFromMeasure IS NULL OR SourceToMeasure IS NULL """
SelectLayerByAttribute(fcAsFeatureLayerLG, NEW_SELECTION_CONST,
selectionQueryL2)
CopyFeatures_management(fcAsFeatureLayerLG, preRouteSourceNoMeasures)
selectionQueryL3 = """ SourceFromMeasure IS NULL """
SelectLayerByAttribute(fcAsFeatureLayerLG, NEW_SELECTION_CONST,
selectionQueryL3)
CalculateField_management(fcAsFeatureLayerLG, "SourceFromMeasure", "0",
PYTHON_9_3_CONST)
selectionQueryL4 = """ SourceToMeasure IS NULL """
SelectLayerByAttribute(fcAsFeatureLayerLG, NEW_SELECTION_CONST,
selectionQueryL4)
CalculateField_management(fcAsFeatureLayerLG, "SourceToMeasure",
"!SHAPE.LENGTH@MILES!", PYTHON_9_3_CONST)
def AnnualizeData(YearToAnnualize):
annualLayer = gdb + "\KTRIPS.SDE.Ktrips_Annual"
currentyYear = gdb + "\KTRIPS.SDE.Ktrips_CurrentYear"
SelectYear = YearSelTest
CalcYear = str(int(YearSelTest - 1))
YearSelTest = "TripYear = '" + SelectYear + "'"
if Exists("Check1"):
Delete_management("Check1")
MakeFeatureLayer_management(annualLayer, "Check1", YearSelTest)
CheckExistence = GetCount_management("Check1")
print CheckExistence
if int(str(CheckExistence)) > 0:
print "This source file info is already in the target feature"
runnext = False
elif int(str(CheckExistence)) == 0:
print 'the information is new for this source file and will be added.'
runnext = True
Append_management(currentyYear, annualLayer, "NO_TEST", "#")
CalculateField_management(annualLayer, "TripYear", CalcYear,
"PYTHON_9.3")
TruncateTable_management(currentyYear)
else:
print 'something isnt working here'
print runnext
pass
def routesSourceCreation():
env.workspace = returnGDBOrSDEPath(routesSourceCenterlines)
env.overwriteOutput = 1
# Checking to see if the output already exists.
# If so, remove it.
if Exists(routesSourceCenterlines):
Delete_management(routesSourceCenterlines)
else:
pass
# Create a new file for the output.
print("Making a copy of " + returnFeatureClass(inputCenterlines) + " called " + returnFeatureClass(routesSourceCenterlines) + ".")
CopyFeatures_management(inputCenterlines, routesSourceCenterlines)
print("Adding fields to " + returnFeatureClass(routesSourceCenterlines) + ".")
#Addfields:
# SourceRouteId (Text, 50)
AddField_management(routesSourceCenterlines, "SourceRouteId", "TEXT", "", "", 50, "SourceRouteId", nullable)
# SourceFromMeasure (Double)
AddField_management(routesSourceCenterlines, "SourceFromMeasure", "DOUBLE", "", "", "", "SourceFromMeasure", nullable)
# SourceToMeasure (Double)
AddField_management(routesSourceCenterlines, "SourceToMeasure", "DOUBLE", "", "", "", "SourceToMeasure", nullable)
if useNewFieldLogic == True:
KDOTKeyCalculation_NewFieldLogic()
else:
TranscendFieldCalculation()
TranscendRampReplacement()
if useNewFieldLogic == True:
KDOTKeyCalculation_NewFieldLogic()
else:
TranscendFieldCalculation()
LocalRouteReduction()
class ACCESSPERMDev(object):
print "access permit points: GO "+ str(datetime.datetime.now())
srcdb =r'ATLASPROD.odc'
if Exists(r'Database Connections/'+srcdb):
Delete_management(DCL+r'\\Database Connections\\'+srcdb)
shutil.copy(conns+"/"+srcdb, DCL+"/"+srcdb)
srcschema = 'KDOT'
srctbl ='KGATE_ACCESSPOINTS_TEST'
source = r'Database Connections/'+srcdb +'/'+srcschema+'.'+srctbl
print source
Lat = "GPS_LATITUDE"
Long = "GPS_LONGITUDE"
loaded = "LOAD_DATE"
dstdb = 'GISTEST.sde'
if Exists(r'Database Connections/'+dstdb):
print dstdb +" exists"
pass
else:
shutil.copy(conns+"/"+dstdb, DCL+"/"+dstdb)
dstschema = 'SHARED'
dstfc = 'ACCESS_POINTS'
dst = r'Database Connections/'+dstdb+'/'+dstschema+'.'+dstfc
print source
print dst
GCS = "GEOGCS['GCS_North_American_1983',DATUM['D_North_American_1983',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119521E-09;0.001;0.001;IsHighPrecision"
XYFC(source, dst, Lat, Long, GCS, loaded)
def setupEnv():
print "run at "+ str(datetime.datetime.now())
rsel = "ENDDATE IS NULL"
MakeTableView_management(resolve, "CCL_Resolution_tbl", rsel)
CalculateField_management("CCL_Resolution_tbl", "CCL_LRS", 'str(!CITYNUMBER!)+str(!LRS_KEY![3:14])', "PYTHON" )
MakeTableView_management(connection1+"CCL_Resolution", "CCL_Resolution_tbl10", 'CITYNUMBER<100')
CalculateField_management("CCL_Resolution_tbl10", "CCL_LRS", '"0"+str(!CITYNUMBER!)+str(!LRS_KEY![3:14])', "PYTHON")
MakeFeatureLayer_management(cntyroutelyr, "cmlrs")
MakeFeatureLayer_management(stateroutelyr, "smlrs")
MakeFeatureLayer_management(citylimits, "CityLimits", "TYPE IN ( 'CS', 'ON')")
LocateFeaturesAlongRoutes_lr(citylimits,"cmlrs","LRS_KEY","0 Feet",connection1+"GIS_CITY","LRS_KEY LINE Beg_CMP End_CMP","FIRST","DISTANCE","NO_ZERO","FIELDS","M_DIRECTON")
MakeRouteEventLayer_lr("cmlrs","LRS_KEY","CCL_Resolution_tbl","LRS_KEY LINE BEG_CNTY_LOGMILE END_CNTY_LOGMILE","City_Connecting_Links","#","ERROR_FIELD","NO_ANGLE_FIELD","NORMAL","ANGLE","LEFT","POINT")
MakeTableView_management(connection1+"GIS_CITY", "GIS_CITY")
MakeTableView_management(laneclass, "LaneClass")
MakeRouteEventLayer_lr("cmlrs","LRS_KEY","GIS_CITY","LRS_KEY LINE BEG_CMP END_CMP","GIS_BASED_CCL","#","ERROR_FIELD","NO_ANGLE_FIELD","NORMAL","ANGLE","LEFT","POINT")
OverlayRouteEvents_lr(connection1+"CCL_Resolution","LRS_KEY LINE BEG_CNTY_LOGMILE END_CNTY_LOGMILE",laneclass,"LRS_KEY LINE BCMP ECMP","INTERSECT",connection1+"CCL_LANE_CLASS_OVERLAY","LRS_KEY LINE BEG_CNTY_LOGMILE END_CNTY_LOGMILE","NO_ZERO","FIELDS","INDEX")
print "create Route Layer specific to City Connecting Link locations"
FeatureClassToFeatureClass_conversion("City_Connecting_Links", connection0, "CITY_CONNECTING_LINK_CENTERLINE")
LocateFeaturesAlongRoutes_lr(connection1+"CITY_CONNECTING_LINK_CENTERLINE",stateroutelyr,"LRS_ROUTE","0 Meters",connection1+"CCL_STATE_LRS_tbl","LRS_ROUTE LINE BEG_STATE_LOGMILE END_STATE_LOGMILE","FIRST","DISTANCE","ZERO","FIELDS","M_DIRECTON")
MakeRouteEventLayer_lr("smlrs", "LRS_ROUTE",connection1+"CCL_STATE_LRS_tbl","LRS_ROUTE LINE BEG_STATE_LOGMILE END_STATE_LOGMILE","CCL_STATE_LRS","#","ERROR_FIELD","NO_ANGLE_FIELD","NORMAL","ANGLE","LEFT","POINT")
FeatureClassToFeatureClass_conversion("CCL_STATE_LRS", connection0, "CITY_CONNECTING_LINK_STATEREF")
if Exists(connection1+"CITY_CONNECTING_LINK_STATE"):
Delete_management(connection1+"CITY_CONNECTING_LINK_STATE")
Dissolve_management(connection1+"CITY_CONNECTING_LINK_STATEREF",connection1+"CITY_CONNECTING_LINK_STATE","LRS_ROUTE;CITY;CITYNUMBER;DESCRIPTION;CCL_LRS","BEG_STATE_LOGMILE MIN;END_STATE_LOGMILE MAX","MULTI_PART","UNSPLIT_LINES")
Dissolve_management(connection1+"CITY_CONNECTING_LINK_STATEREF",connection1+"CITY_CONNECTING_LINK_STATE_D","CCL_LRS","BEG_STATE_LOGMILE MIN;END_STATE_LOGMILE MAX","MULTI_PART","DISSOLVE_LINES")
print "processes to Create the layer that will be used to create a new LRS for city connecting links"
def XYFC(source, dst, Lat, Long, GCS, loaded):
if Exists("FCtbl"):
Delete_management("FCtbl")
else:
pass
if Exists("FC_Layer"):
Delete_management("FC_Layer")
else:
pass
print "start XYFC "+ str(datetime.datetime.now())
MakeTableView_management(source, 'FCtbl', "#", "#", "")
MakeXYEventLayer_management("FCtbl",Long, Lat,"FC_Layer", GCS,"#")
TruncateTable_management(dst)
Append_management("FC_Layer",dst,"NO_TEST","#","#")
CalculateField_management(dst, loaded,"datetime.datetime.now( )","PYTHON_9.3","#")
print "XYFC complete for " +str(dst)+ " at " + str(datetime.datetime.now())
def AddInsert(fc, layer_name, newtbl, workspace):
MakeTableView_management(newtbl, "NEWROWS_View", "#", "#", "#")
addcount = int(GetCount_management("NEWROWS_View").getOutput(0))
if addcount == 0:
print "no new records"
pass
else:
MakeFeatureLayer_management(fc, layer_name)
MakeXYEventLayer_management(
"NEWROWS_View", "CROSSINGLONGITUDE", "CROSSINGLATITUDE",
"NEWROWS_Layer",
"GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],VERTCS['NAVD_1988',VDATUM['North_American_Vertical_Datum_1988'],PARAMETER['Vertical_Shift',0.0],PARAMETER['Direction',1.0],UNIT['Meter',1.0]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119522E-09;0.001;0.001;IsHighPrecision",
"#")
FeatureClassToFeatureClass_conversion(
"NEWROWS_Layer", "D:/Temp", "LOADTHIS1.shp", "#",
"""CROSSINGID "CROSSINGID" true false false 30 Text 0 0 ,First,#,Database Connections/sdedev_ciims.sde/CIIMS.NEWROWS_Features,CROSSINGID,-1,-1;CROSSINGLA "CROSSINGLA" true true false 8 Double 10 38 ,First,#,Database Connections/sdedev_ciims.sde/CIIMS.NEWROWS_Features,CROSSINGLATITUDE,-1,-1;CROSSINGLO "CROSSINGLO" true true false 8 Double 10 38 ,First,#,Database Connections/sdedev_ciims.sde/CIIMS.NEWROWS_Features,CROSSINGLONGITUDE,-1,-1;CROSSINGTY "CROSSINGTY" true true false 2 Text 0 0 ,First,#,Database Connections/sdedev_ciims.sde/CIIMS.NEWROWS_Features,CROSSINGTYPE,-1,-1""",
"#")
Append_management(
"D:/Temp/LOADTHIS1.shp", layer_name, "NO_TEST",
"""CROSSINGID "CROSSINGID" true false false 30 Text 0 0 ,First,#,D:/Temp/LOADTHIS1.shp,CROSSINGID,-1,-1;CROSSINGLATITUDE "CROSSINGLATITUDE" true true false 8 Double 10 38 ,First,#,D:/Temp/LOADTHIS1.shp,CROSSINGLA,-1,-1;CROSSINGLONGITUDE "CROSSINGLONGITUDE" true true false 8 Double 10 38 ,First,#,D:/Temp/LOADTHIS1.shp,CROSSINGLO,-1,-1;CROSSINGTYPE "CROSSINGTYPE" true true false 2 Text 0 0 ,First,#,D:/Temp/LOADTHIS1.shp,CROSSINGTY,-1,-1;LOADDATE "LOADDATE" true true false 36 Date 0 0 ,First,#""",
"#")
Delete_management("D:/Temp/LOADTHIS1.shp", "#")
updatelyr = layer_name + "new"
MakeFeatureLayer_management(layer_name, updatelyr, "LOADDATE IS NULL")
with da.Editor(workspace) as edit:
CalculateField_management(updatelyr, "LOADDATE",
"datetime.datetime.now( )", "PYTHON_9.3",
"#")
del fc, layer_name, newtbl, workspace, updatelyr
print "new rows inserted into Static_Crossings"
def RemoveGpHistory_fc(out_xml_dir):
remove_gp_history_xslt = r"C:\GIS\metadataremoval\removeGeoprocessingHistory.xslt"
print "Trying to remove out_xml_dir/metadtaTempFolder..."
if Exists(out_xml_dir):
Delete_management(out_xml_dir)
else:
pass
os.mkdir(out_xml_dir)
env.workspace = out_xml_dir
ClearWorkspaceCache_management()
try:
print "Starting xml conversion."
name_xml = "CMLRS_LAM.xml"
#Process: XSLT Transformation
XSLTransform_conversion(gdb_93_CMLRS, remove_gp_history_xslt, name_xml, "")
print("Completed xml conversion on %s") % (gdb_93_CMLRS)
# Process: Metadata Importer
MetadataImporter_conversion(name_xml, gdb_93_CMLRS)
except:
print("Could not complete xml conversion on %s") % (gdb_93_CMLRS)
endTime = datetime.datetime.now()
ScriptStatusLogging('Cansys_CMLRS_Transfer', 'SharedSDEProd.gdb\SHARED_CANSYS_CMLRS',
scriptFailure, startTime, endTime, "Could not complete xml conversion on " + gdb_93_CMLRS,
pythonLogTable)
# Reraise the error to stop execution and prevent a success message
# from being inserted into the table.
raise
def ImportAllSheets(inExcel):
workbook = xlrd.open_workbook(inExcel)
sheets = [sheet.name for sheet in workbook.sheets()]
print('{} sheets found: {}'.format(len(sheets), ','.join(sheets)))
sheetCounter = 0
for sheet in sheets:
# The out_table is based on the input excel file name
# an underscore (_) separator followed by the sheet name
if sheetCounter == 0:
try:
try:
Delete_management(countyMapSizes)
except:
pass
print('Converting {} to {}'.format(sheet, countyMapSizes))
# Perform the conversion
ExcelToTable_conversion(inExcel, countyMapSizes, sheet)
except:
print "There was an error in writing the countyMapSizes table."
else:
pass
sheetCounter = sheetCounter + 1
print "Sheet import complete!"
def setupQCGDB():
print("Setting up the QC GDB.")
if (Exists(errorFeaturesQCGDB)):
Delete_management(errorFeaturesQCGDB)
else:
pass
CreateFileGDB_management(mainFolder, errorFeaturesQCGDBName)
class CIIMSDev(object):
srcdb =r'sdedev_ciims.sde'
if Exists(r'Database Connections/'+srcdb):
Delete_management(DCL+r'\\Database Connections\\'+srcdb)
shutil.copy(conns+"/"+srcdb, DCL+"/"+srcdb)
srcschema = 'CIIMS'
srctbl ='CIIMS_VWCROSSINGGIS3'
source = r'Database Connections/'+srcdb +'/'+srcschema+'.'+srctbl
Lat = "CROSSINGLATITUDE"
Long = "CROSSINGLONGITUDE"
loaded = "LOADDATE"
dstdb = r'sdedev_ciims.sde'
if Exists(r'Database Connections/'+dstdb):
pass
else:
shutil.copy(conns+"/"+dstdb, DCL+"/"+dstdb)
dstschema = 'CIIMS'
dstfd = 'CIIMS'
dstfc = 'Static_Crossings'
dst = r'Database Connections/'+dstdb+'/'+dstschema+'.'+dstfd+'/'+dstschema+'.'+dstfc
GCS = "GEOGCS['GCS_North_American_1983',DATUM['D_North_American_1983',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119521E-09;0.001;0.001;IsHighPrecision"
XYFC(source, dst, Lat, Long, GCS, loaded)
def calculateMeasuresForLocalRoutes(routesToMeasure, subsetSelectionQuery):
# Make a feature layer
# Select it with the subsetSelectionQuery
# If the number of selected features is at least 1
# Then, run the calculateField_management calls for
# the selected features.
fcAsFeatureLayerForMeasuring = 'FCAsFeatureLayer_Measures'
if Exists(fcAsFeatureLayerForMeasuring):
Delete_management(fcAsFeatureLayerForMeasuring)
else:
pass
MakeFeatureLayer_management(routesToMeasure, fcAsFeatureLayerForMeasuring)
SelectLayerByAttribute_management(fcAsFeatureLayerForMeasuring,
'CLEAR_SELECTION')
SelectLayerByAttribute_management(fcAsFeatureLayerForMeasuring,
'NEW_SELECTION', subsetSelectionQuery)
countResult = GetCount_management(fcAsFeatureLayerForMeasuring)
intCount = int(countResult.getOutput(0))
print('There were ' + str(intCount) +
' features selected in the fcAsFeatureLayerForMeasuring layer.')
if intCount >= 1:
expressionText1 = 0
CalculateField_management(fcAsFeatureLayerForMeasuring, startMeasure,
expressionText1, "PYTHON_9.3")
expressionText2 = 'float("{0:.3f}".format(!Shape_Length! / 5280.00))'
CalculateField_management(fcAsFeatureLayerForMeasuring, endMeasure,
expressionText2, "PYTHON_9.3")
else:
print "Not calculating due to lack of selected features."
def moveLocalErrorsToSQL(prefixKeyName):
print("Moving errors from the Local gdb to SQL for the prefix key name of: " + str(prefixKeyName) + ".")
for errorItemFC in prefixesToMoveErrorsFor:
errorItemFCBaseName = returnFeatureClass(errorItemFC)
sqlPath = returnGDBOrSDEPath(dailyProcessSDESourceCenterlinesLocation)
errorItemCopyName = prefixKeyName + '_' + errorItemFCBaseName
errorItemSQLLocation = os.path.join(sqlPath, errorItemCopyName)
lowerstrErrorItemFCBaseName = str(errorItemFCBaseName).lower()
if lowerstrErrorItemFCBaseName.find('selfintclassification') >= 0:
# Already includes the prefix name in the basename, so just use the full BaseName for the class.
errorItemSQLLocation = os.path.join(sqlPath, errorItemFCBaseName)
else:
pass
if Exists(errorItemSQLLocation):
try:
Delete_management(errorItemSQLLocation)
except:
print("Could not delete the FC at: " + str(errorItemSQLLocation) + ".")
print("Please make sure that the FC does not have any locks on it and try again.")
try:
CopyFeatures_management(errorItemFC, errorItemSQLLocation)
except:
print("Could not copy from the FC at: " + str(errorItemFC))
print("to the FC at: " + str(errorItemSQLLocation) + ".")
print("Please make sure that the FC to copy from exists")
print("and that the FC to copy to is not locked.")
def RemoveGpHistory_fc(out_xml_dir):
remove_gp_history_xslt = r"D:\kandrive\harvesters\scheduled-tasks\metadataremoval\removeGeoprocessingHistory.xslt"
print "Trying to remove out_xml_dir/metadtaTempFolder..."
if Exists(out_xml_dir):
Delete_management(out_xml_dir)
else:
pass
os.mkdir(out_xml_dir)
env.workspace = out_xml_dir
ClearWorkspaceCache_management()
try:
print "Starting xml conversion."
name_xml = "RCRS_LAM.xml"
#Process: XSLT Transformation
XSLTransform_conversion(kanDriveSpatialConditions,
remove_gp_history_xslt, name_xml, "")
print("Completed xml conversion on %s") % (kanDriveSpatialConditions)
# Process: Metadata Importer
MetadataImporter_conversion(name_xml, kanDriveSpatialConditions)
except:
print("Could not complete xml conversion on %s") % (
kanDriveSpatialConditions)
endTime = datetime.datetime.now()
ScriptStatusLogging(
'KanDrive_Spatial_Conditions_Update',
'kandrive_spatial.DBO.Conditions', scriptFailure, startTime,
endTime, "Could not complete xml conversion on " +
kanDriveSpatialConditions, pythonLogTable)
# Reraise the error to stop execution and prevent a success message
# from being inserted into the table.
raise
def UpdateLocalFileGDB():
import datetime, time
fDateTime = datetime.datetime.now()
from arcpy import FeatureClassToFeatureClass_conversion, CreateFileGDB_management, Exists, Delete_management
from KhubCode25.KhubCode25Config import (
localProProjectPath, localProFileGDBWorkspace, prodDataSourceSDE,
devDataSourceSDE, dbname, dbownername, countylines, devorprod)
if devorprod == 'prod':
database = prodDataSourceSDE
print("running on " + devorprod)
else:
database = devDataSourceSDE
print("running on " + devorprod)
fileformatDateStr = fDateTime.strftime("%Y%m%d")
localfilegdb = localProFileGDBWorkspace + '\\' + 'KhubRoadCenterlines' + fileformatDateStr + '.gdb'
#print(fileformatDateStr)
if Exists(localfilegdb):
print(localfilegdb + " exists and will be deleted")
Delete_management(localfilegdb)
time.sleep(1)
CreateFileGDB_management(localProFileGDBWorkspace,
"KhubRoadCenterlines" + fileformatDateStr,
"CURRENT")
FeatureClassesUsed = [
'All_Road_Centerlines', 'All_Road_Centerlines_D1', 'MARKUP_POINT',
'All_Roads_Stitch_Points', 'Videolog_CURRENT_LANETRACE',
'Videolog_CURRENT_RAMPTRACE', 'HPMS_RAMPS'
]
for FeatureClass in FeatureClassesUsed:
loopFC = localProProjectPath + '/' + database + "/" + dbname + "." + dbownername + "." + FeatureClass
FeatureClassToFeatureClass_conversion(loopFC, localfilegdb,
FeatureClass)
FeatureClassToFeatureClass_conversion(
localProProjectPath + '/' + countylines, localfilegdb,
"SHARED_COUNTY_LINES")
def FileChecker(ShapeFileDate, CheckLayer):
#runnext = False
SourceFileTxt = str(ShapeFileDate.replace("-", "_"))
print ShapeFileDate
selectdate = "Sourcefile = '" + SourceFileTxt + "'"
print selectdate
print CheckLayer
if Exists("Check1"):
Delete_management("Check1")
try:
MakeFeatureLayer_management(CheckLayer, "Check1", selectdate)
except:
MakeTableView_management(CheckLayer, "Check1", selectdate)
CheckExistence = GetCount_management("Check1")
print CheckExistence
if int(str(CheckExistence)) > 0:
print "This source file info is already in the target feature"
runnext = False
elif int(str(CheckExistence)) == 0:
print 'the information is new for this source file and will be added.'
runnext = True
else:
print 'something isnt working here'
print runnext
return runnext
def ReturnStreetstoTopology():
from arcpy import (AddFeatureClassToTopology_management, ListDatasets,
AddRuleToTopology_management, Delete_management,
Describe)
env.workspace = currentPathSettings.gdbPath
fd = ListDatasets("*", "Feature")
gdbw = os.path.join(currentPathSettings.gdbPath, fd[0])
env.workspace = gdbw
topoDatasetList = ListDatasets("*", "TOPOLOGY")
geonetDatasetList = ListDatasets("*", "GeometricNetwork")
authbnd = os.path.join(gdbw, "AuthoritativeBoundary")
ESZBnd = os.path.join(gdbw, "ESZ")
if geonetDatasetList == []:
print "no geometric network created yet"
else:
Delete_management(geonetDatasetList[0])
desc = Describe(topoDatasetList[0])
print "%-27s %s" % ("FeatureClassNames:", desc.featureClassNames)
if lyr in desc.featureClassNames:
print "Road Centerlines already exist in topology dataset"
else:
print "adding road centerlines to topology"
inputTopology = os.path.join(gdbw, topoDatasetList[0])
inputRoadCenterline = os.path.join(gdbw, "RoadCenterline")
AddFeatureClassToTopology_management(inputTopology,
inputRoadCenterline, "1", "1")
AddRuleToTopology_management(inputTopology, "Must Not Overlap (line)",
inputRoadCenterline, "", "", "")
AddRuleToTopology_management(inputTopology,
"Must Not Intersect (line)",
inputRoadCenterline, "", "", "")
AddRuleToTopology_management(inputTopology,
"Must Not Have Dangles (Line)",
inputRoadCenterline, "", "", "")
AddRuleToTopology_management(inputTopology,
"Must Not Self-Overlap (Line)",
inputRoadCenterline, "", "", "")
AddRuleToTopology_management(inputTopology,
"Must Not Self-Intersect (Line)",
inputRoadCenterline, "", "", "")
AddRuleToTopology_management(inputTopology,
"Must Be Single Part (Line)",
inputRoadCenterline, "", "", "")
AddRuleToTopology_management(
inputTopology, "Must Not Intersect Or Touch Interior (Line)",
inputRoadCenterline, "", "", "")
AddRuleToTopology_management(
inputTopology, "Must Not Intersect Or Touch Interior (Line)",
inputRoadCenterline, "", "", "")
AddRuleToTopology_management(inputTopology,
"Must Be Inside (Line-Area)",
inputRoadCenterline, "", authbnd, "")
#write for loop - must be inside ESZ boundaries
AddRuleToTopology_management(
inputTopology, "Boundary Must Be Covered By (Area-Line)", authbnd,
"", inputRoadCenterline, "")
AddRuleToTopology_management(
inputTopology, "Boundary Must Be Covered By (Area-Line)", ESZBnd,
"", inputRoadCenterline, "")
def Maintenance():
print "reference maintenance agreement table"
MakeTableView_management(maintenance, "Maint_tview")
MakeRouteEventLayer_lr(cntyroutelyr,"LRS_KEY", "Maint_tview","LRSKEY LINE BEGMILEPOST END_MP","Maintenance_Events_CNTY","#","ERROR_FIELD","ANGLE_FIELD","NORMAL","ANGLE","LEFT","POINT")
if Exists(connection1+"MAINTENANCE_CCL"):
Delete_management(connection1+"MAINTENANCE_CCL")
LocateFeaturesAlongRoutes_lr("Maintenance_Events_CNTY",connection1+"CCL_LRS_ROUTE",NewRouteKey,"1 Feet",connection1+"MAINTENANCE_CCL","CCL_LRS LINE CCL_BEGIN CCL_END","ALL","DISTANCE","ZERO","FIELDS","M_DIRECTON")
print "show lane classification referenced to city connecting link LRS"
def DirectionalUrbanClass():
#do the same as for State Sys but for Non State Urban Classified Highways (Nusys)
FeatureVerticesToPoints_management("RoadCenterlinesC",
"in_memory/UrbanPoints", "MID")
FeatureClassToFeatureClass_conversion(
NSND,
"in_memory",
"NSND_NPD",
where_clause="NETWORK_DIRECTION IN ( 'SB' , 'WB' )")
LocateFeaturesAlongRoutes_lr("UrbanPoints", "NSND_NPD", "NE_UNIQUE",
"200 Feet", "in_memory/UrbanPointsMeasures",
"RID POINT MEAS", "ALL", "DISTANCE", "ZERO",
"FIELDS", "M_DIRECTON")
Delete_management("UrbanPoints")
#is the query stil valid for non state system? Explore hte NE Unique. State System did not use the county number prefix
SelectLayerByAttribute_management(
"UrbanPointsMeasures", "NEW_SELECTION",
"""SUBSTRING( "RID", 4, 7) NOT LIKE SUBSTRING("NON_State_System_LRSKey" ,4, 7)"""
)
DeleteRows_management(in_rows="UrbanPointsMeasures")
MakeRouteEventLayer_lr("NSND_NPD",
"NE_UNIQUE",
"UrbanPointsMeasures",
"rid POINT MEAS",
"UrbanPointEvents",
offset_field="Distance",
add_error_field="ERROR_FIELD",
add_angle_field="ANGLE_FIELD",
angle_type="NORMAL",
complement_angle="ANGLE",
offset_direction="RIGHT",
point_event_type="POINT")
MakeFeatureLayer_management("UrbanPointEvents", "UNPD_ID",
""""Distance">=0""")
SelectLayerByLocation_management("UNPD_ID", "INTERSECT",
"RoadCenterlinesC", "1 Feet",
"NEW_SELECTION")
#at this point, there are a lot of false positives, places with single carriagway roads and nusys divided
#we need to incorporate the check overlap process to identify where their are single and dual carriagways here
#starting with the technique to find sausages or dual carraigeways
#SpatialJoin_analysis("UNPD_ID", "RoadCenterlinesC", "in_memory/ValidateSausages120", "JOIN_ONE_TO_ONE", "KEEP_ALL", '#', "INTERSECT", "120 Feet", "Distance")
#this Spatial Join step is improving the results, removing most false positives. It still shows overlapping segments
#it would be improved even more, potentially, by testing the non-primary direction against dissolve somehow.
#except, the calculate method is by segment to the source, a dissolve would complicate the process of calculating back to the source data
#we are looking for count grater than 0 of the offset point to hte segment, so a dissolved segment should work
import EliminateOverlaps
from EliminateOverlaps import CollectorDissolve
#set the roadcenterline input and dissolve output for RoadCenterline dissolve for this subroutine
roadcenterlines = "RoadCenterlinesC"
ClassOutput = r"in_memory/RMC2"
CollectorDissolve()
SpatialJoin_analysis("UNPD_ID", "RMC2dissolve",
"in_memory/ValidateSausages120", "JOIN_ONE_TO_ONE",
"KEEP_ALL", '#', "INTERSECT", "120 Feet", "Distance")
def AnnualStats(ShapeFileDate):
env.overwriteOutput = 1
#SourceFileTxt = str(ShapeFileDate[12:-4].replace("-", ""))
#infileMonthly = yeardb+r"\Kansas\LRS"+SourceFileTxt
qyear = ShapeFileDate[16:20]
#try:
# Delete_management(yeardb+"/KTRIPS_MonthlySum_Statistics")
# Delete_management(yeardb+"/RunningTotal")
#except:
# print "nothing deleted"
#sumfile = gdb+"\INTERMODAL.DBO.KTRIPS_MonthlySum"
sumfile = gdb + "\KTRIPS.SDE.KTRIPS_MonthlySum"
whereclause = str(BuildWhereClauseLike(sumfile, "SourceFile", qyear))
if Exists("ThisYearMonthly"):
Delete_management("ThisYearMonthly")
MakeFeatureLayer_management(sumfile, "ThisYearMonthly", whereclause, "#",
"#")
Statistics_analysis("ThisYearMonthly",
"in_memory/KTRIPS_MonthlySum_Statistics",
"Join_Count SUM;Tonnage SUM",
"LRS_KEY;BEG_CNTY_LOGMILE;END_CNTY_LOGMILE")
AddField_management("in_memory/KTRIPS_MonthlySum_Statistics", "TonMiles",
"DOUBLE", "#", "#", "#", "#", "NULLABLE",
"NON_REQUIRED", "#")
CalculateField_management(
"in_memory/KTRIPS_MonthlySum_Statistics", "TonMiles",
"!SUM_Tonnage! /(!END_CNTY_LOGMILE! - !BEG_CNTY_LOGMILE!)",
"PYTHON_9.3", "#")
if Exists("KTRIPS_RunningTotal_CurentYear"):
Delete_management("KTRIPS_RunningTotal_CurentYear")
MakeRouteEventLayer_lr(cansys, "LRS_KEY",
"in_memory/KTRIPS_MonthlySum_Statistics",
"LRS_KEY LINE BEG_CNTY_LOGMILE END_CNTY_LOGMILE",
"KTRIPS_RunningTotal_CurentYear", "#",
"ERROR_FIELD", "NO_ANGLE_FIELD", "NORMAL", "ANGLE",
"LEFT", "POINT")
CurrentYrStat = gdb + "\KTRIPS.SDE.Ktrips_CurrentYear"
TruncateTable_management(CurrentYrStat)
Append_management("KTRIPS_RunningTotal_CurentYear", CurrentYrStat,
"NO_TEST", "#", "")
print "annual Stats have been recalculated from the latest Monthly Statistics"
Delete_management("in_memory/KTRIPS_MonthlySum_Statistics")
def addProperty(self, in_raster):
"""
get value of a field and write it to a column named RASTERVALU in the object
@param in_raster: raster where the value is taken from
"""
desc = Describe(self.filepath)
name = "addProperty" + str(self.sObj)
outputLocation = "in_memory\\" + name
if desc.shapeType == "Point":
ExtractValuesToPoints(self.filepath, in_raster.filepath,
outputLocation)
addProperty = utils.makeObject(outputLocation)
elif desc.shapeType == "Line":
raise NotImplementedError(desc.shapeType)
elif desc.shapeType == "Polygon":
polyToPoint = "in_memory\\polyToPoint_" + str(self.sObj)
FeatureToPoint_management(self.filepath, polyToPoint, "CENTROID")
valueToPoint = "in_memory\\valueToPoint_" + str(self.sObj)
ExtractValuesToPoints(polyToPoint, in_raster.filepath,
valueToPoint)
CopyFeatures_management(self.filepath, outputLocation)
JoinField_management(outputLocation, "FID", valueToPoint, "FID",
"RASTERVALU")
addProperty = utils.makeObject(outputLocation)
Delete_management(polyToPoint)
Delete_management(valueToPoint)
#TODO implement method that the parameters "CENTROID" or "INSIDE" for FeatureToPoint_management() can be selected
else:
raise NotImplementedError("unknown shapeType:", desc.shapeType)
# update cc instance's attributes
desc = Describe(outputLocation)
addProperty.domain = desc.extent
addProperty.filepath = outputLocation
addProperty.filename = os.path.basename(outputLocation)
return addProperty
def checkAddressPointFrequency(AddressPoints, gdb):
from arcpy import Frequency_analysis, MakeTableView_management, DeleteRows_management, GetCount_management, Delete_management, Exists
from os.path import join
AP_Freq = join(gdb, "AP_Freq")
fl = "fl"
#remove the frequency table if it exists already
if Exists(AP_Freq):
Delete_management(AP_Freq)
#run frequency analysis
Frequency_analysis(
AddressPoints, AP_Freq,
"MUNI;HNO;HNS;PRD;STP;RD;STS;POD;POM;ZIP;BLD;FLR;UNIT;ROOM;SEAT;LOC;LOCTYPE",
"")
#get count of records
rFreq = GetCount_management(AP_Freq)
rCount = int(rFreq.getOutput(0))
#delete records
#make where clause
wc = "Frequency = 1 or LOCTYPE <> 'Primary'"
#make feature layer
MakeTableView_management(AP_Freq, fl, wc)
#get count of the results
result = GetCount_management(fl)
count = int(result.getOutput(0))
if rCount != count:
#Delete
DeleteRows_management(fl)
userMessage(
"Checked frequency of address points. Results are in table " +
AP_Freq)
elif rCount == count:
Delete_management(AP_Freq)
userMessage("All address points are unique.")
def GetRefposts():
from arcpy import WFSToFeatureClass_conversion, CreateFileGDB_management, Delete_management
input_WFS_server = r"http://wfs.ksdot.org/arcgis_web_adaptor/services/Structures/Reference_Post_Signs/MapServer/WFSServer?request=GetCapabilities&service=WFS"
gdbin = GDB_In + ".gdb"
try:
CreateFileGDB_management(r"C:/temp", gdbname, "CURRENT")
print "new created " + gdbin
except:
Delete_management(gdbin, "Workspace")
CreateFileGDB_management(r"C:/temp", gdbname, "CURRENT")
print "refreshed " + gdbin
WFSToFeatureClass_conversion(input_WFS_server, fcname, gdbin, fcname)
def recreateLoganProcessAdditionsGDB():
print("Starting the recreateLoganProcessAdditionsGDB function!")
if Exists(loganProcessAdditionsGDB):
try:
Delete_management(loganProcessAdditionsGDB)
except:
print("Could not delete the loganProcessAdditionsGDB.")
print("Please remove any locks and try again.")
else:
pass
CreateFileGDB_management(mainFolder, loganProcessAdditionsGDBName)
def copyFCToTempLocation(inputFC, outputFC):
if Exists(outputFC):
try:
Delete_management(outputFC)
except:
print("Could not delete the temp FC at: " + str(outputFC) + ".")
else:
pass
try:
CopyFeatures_management(inputFC, outputFC)
except:
print("Could not copy the feature class at: " + str(inputFC) +
" to the output location at: " + str(outputFC) + ".")
def checkFeatureLocations(gdb):
userMessage("Checking feature locations...")
from os import path
from arcpy import MakeFeatureLayer_management, SelectLayerByAttribute_management, SelectLayerByLocation_management, GetCount_management, Delete_management, da
values = []
#make sure feature are all inside authoritative boundary
#get authoritative boundary
authBound = path.join(gdb, "NG911", "AuthoritativeBoundary")
ab = "ab"
MakeFeatureLayer_management(authBound, ab)
for dirpath, dirnames, filenames in da.Walk(gdb, True, '', False,
["FeatureClass"]):
for filename in filenames:
if filename != "AuthoritativeBoundary":
#get full path name & create a feature layer
fullPath = path.join(gdb, filename)
fl = "fl"
MakeFeatureLayer_management(fullPath, fl)
#select by location to get count of features outside the authoritative boundary
SelectLayerByLocation_management(fl, "INTERSECT", ab)
SelectLayerByAttribute_management(fl, "SWITCH_SELECTION", "")
#get count of selected records
result = GetCount_management(fl)
count = int(result.getOutput(0))
#report results
if count > 0:
fields = ("OBJECTID")
with da.SearchCursor(fl, fields) as rows:
for row in rows:
val = (today,
"Feature not inside authoritative boundary",
filename, "", row[0])
values.append(val)
else:
userMessage(filename +
": all records inside authoritative boundary")
#clean up
Delete_management(fl)
userMessage("Completed check on feature locations")
if values != []:
RecordResults("fieldValues", values, gdb)
