def crs_update_datasource_mxd(args):
# script name
script_name = os.path.basename(__file__)
# script parameters
mxd = args[0]
gdb = args[1]
log = args[2]
err_message = None
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
try:
mxdDoc = arcpy.mapping.MapDocument(mxd)
for lyr in arcpy.mapping.ListLayers(mxdDoc):
# if lyr.isBroken:
lyr.findAndReplaceWorkspacePath(lyr.workspacePath, gdb)
# mxd.findAndReplaceWorkspacePaths(r"C:\Project\Data", r"C:\Project\Data2")
mxdDoc.save()
del mxdDoc
except Exception as e:
err_message = "ERROR while running {0}: {1}" .format(script_name,e)
return err_message
# if __name__ == '__main__':
# main(args)
def crs10_prod_db_refresh (args):
# parameters
prepSPREPORTpath = args[0]
prodSPREPORTpath = args[1]
prepSPOWNpath = args[2]
prodSPOWNpath = args[3]
log = args[4]
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
# variables
err_message = None
try:
##### Truncate data in production SDEs and then append from preprod
### SPREPORT
for fc in spreportFClist:
# Assign locations
inFC = os.path.join(prepSPREPORTpath,fc)
prodFC = os.path.join(prodSPREPORTpath,fc)
# Count input number of records
infcRowCount = arcpy.GetCount_management(inFC).getOutput(0)
etgLib.log_info(log, 'INPUT {0}: {1} records'.format(inFC,infcRowCount))
# Truncate
arcpy.TruncateTable_management(prodFC)
# Append
arcpy.Append_management(inFC,prodFC,"NO_TEST")
# Count number of records in prod
prodfcRowCount = arcpy.GetCount_management(prodFC).getOutput(0)
etgLib.log_info(log, 'PRODUCTION {0}: {1} records'.format(prodFC,prodfcRowCount))
### SPOWN
for fc in spownFClist:
# Assign locations
inFC = os.path.join(prepSPOWNpath,fc)
prodFC = os.path.join(prodSPOWNpath,fc)
# Count input number of records
infcRowCount = arcpy.GetCount_management(inFC).getOutput(0)
etgLib.log_info(log, 'INPUT {0}: {1} records'.format(inFC,infcRowCount))
# Truncate
arcpy.TruncateTable_management(prodFC)
fieldMap = "TP_PROPERTY_LINK_PROPERTY_ID \"TP_PROPERTY_LINK_PROPERTY_ID\" true false false 8 Double 0 0 ,First,#," + inFC + ",PROPERTY_ID,-1,-1;OBJECTID_1 \"OBJECTID_1\" true false false 4 Long 0 0 ,First,#;PROPERTY_ID \"PROPERTY_ID\" true false false 8 Double 0 0 ,First,#," + inFC + ",PROPERTY_ID,-1,-1;PARCEL_ID \"PARCEL_ID\" true false false 8 Double 0 0 ,First,#," + inFC + ",PROPERTY_ID,-1,-1;COUNT_PARCEL_ID \"COUNT_PARCEL_ID\" true true false 2 Short 0 0 ,First,#," + inFC + ",COUNT_PARCEL_ID,-1,-1;SHAPE_STArea__ \"SHAPE_STArea__\" true false true 8 Double 0 0 ,First,#;SHAPE_STLength__ \"SHAPE_STLength__\" true false true 8 Double 0 0 ,First,#;SHAPE_Length \"SHAPE_Length\" false true true 8 Double 0 0 ,First,#;SHAPE_Area \"SHAPE_Area\" false true true 8 Double 0 0 ,First,#"
# Append
arcpy.Append_management(inFC,prodFC,"NO_TEST", fieldMap)
# Count number of records in prod
prodfcRowCount = arcpy.GetCount_management(prodFC).getOutput(0)
etgLib.log_info(log, 'PRODUCTION {0}: {1} records'.format(prodFC,prodfcRowCount))
etgLib.log_process_time(log,starttime)
except Exception as e:
print ("ERROR: {}".format(e))
err_message = "ERROR while running {0}: {1}" .format(script_name,e)
return err_message
def crs9_check_preprod_feature_counts(args):
# parameters
labelsPath = args[0]
preprodPath = args[1]
preprodPrefix = args[2]
stagingPath = args[3]
stagingPrefix = args[4]
log = args[5]
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
# variables
err_message = None
try:
### Get lists of tables and FCs
# Get tables and FCs from labels GDB
etgLib.log_info(log,
'Getting lists of tables and FCs from labels GDB...')
arcpy.env.workspace = labelsPath
fcllblg = arcpy.ListFeatureClasses()
tblllblg = arcpy.ListTables()
# Get tables and FCs from staging SDE
etgLib.log_info(log,
'Getting lists of tables and FCs from staging SDE...')
arcpy.env.workspace = stagingPath
fclstgs = arcpy.ListFeatureClasses()
tbllstgs = arcpy.ListTables()
# Get tables and FCs from preprod SDE
etgLib.log_info(log,
'Getting lists of tables and FCs from preprod SDE...')
arcpy.env.workspace = preprodPath
fclpprs = arcpy.ListFeatureClasses()
tbllpprs = arcpy.ListTables()
### Work through lists of feature classes
etgLib.log_info(log, '==> Checking FC counts for preprod...', True)
arcpy.env.workspace = preprodPath
# Get preprod values
for pprsfc in fclpprs:
# Ignore views and old data
pprsfcname = pprsfc[len(preprodPrefix):]
if pprsfcname.endswith("_1") or pprsfcname.endswith(
"_o") or pprsfcname.endswith("_vw") or pprsfcname.endswith(
"_oo"):
etgLib.log_info(log, '{} - ignoring...'.format(pprsfc))
else:
# Set prelim values
pprsfccount = dummyVal
stgsfccount = dummyVal
lblgfccount = dummyVal
# crsgfccount = dummyVal
# Get preprod count
pprsfccount = arcpy.GetCount_management(pprsfc).getOutput(0)
etgLib.log_info(
log,
'{} - preprod count = {}'.format(pprsfcname, pprsfccount))
# Find comparable staging FC
for stgsfc in fclstgs:
stgsfcname = stgsfc[len(stagingPrefix):]
if stgsfcname == pprsfcname:
# Get staging count
stgsfcpath = os.path.join(stagingPath, stgsfc)
stgsfccount = arcpy.GetCount_management(
stgsfcpath).getOutput(0)
stgsfccountname = stgsfcname
continue
# Report staging count status
if stgsfccount != dummyVal and stgsfccount != pprsfccount:
etgLib.log_info(
log,
'*****ERROR!!!***** preprod count = {0} but staging count = {1}'
.format(pprsfccount, stgsfccount))
elif stgsfccount == dummyVal:
etgLib.log_info(
log, '{} not found in staging SDE'.format(pprsfcname))
else:
etgLib.log_info(
log, '{0} - staging count = {1}'.format(
stgsfccountname, stgsfccount))
# Find comparable labels FC
for lblgfc in fcllblg:
if lblgfc == pprsfcname:
# Get labels count
lblgfcpath = os.path.join(labelsPath, lblgfc)
lblgfccount = arcpy.GetCount_management(
lblgfcpath).getOutput(0)
lblgfccountname = lblgfc
continue
# Report labels count status
if lblgfccount != dummyVal and lblgfccount != pprsfccount:
etgLib.log_info(
log,
'*****ERROR!!!***** preprod count = {0} but labels count = {1}'
.format(pprsfccount, lblgfccount))
elif lblgfccount == dummyVal:
etgLib.log_info(
log, '{} not found in labels SDE'.format(pprsfcname))
else:
etgLib.log_info(
log, '{0} - labels count = {1}'.format(
lblgfccountname, lblgfccount))
### Work through lists of tables
etgLib.log_info(log, '==> Checking table counts for preprod...', True)
# Get preprod values
for pprstbl in tbllpprs:
# Ignore views and old data
pprstblname = pprstbl[len(preprodPrefix):]
if pprstblname.endswith("_o") or pprstblname.endswith("_vw") or pprstblname.endswith("_oo") or \
pprstblname.startswith("mv_") or pprstblname.startswith("vw") or pprstblname.startswith("VW_"):
etgLib.log_info(log, '{} - ignoring...'.format(pprstblname))
else:
# Set prelim values
pprstblcount = dummyVal
stgstblcount = dummyVal
lblgtblcount = dummyVal
# Get preprod count
#pprstblname = pprstbl[len(preprodPrefix):]
if pprstblname in tblIgnoreList:
etgLib.log_info(
log,
'WARNING: ignoring {} ***** manual check required *****'
.format(pprstblname))
continue
else:
pprstblcount = arcpy.GetCount_management(
pprstbl).getOutput(0)
etgLib.log_info(
log, '{} - preprod count = {}'.format(
pprstblname, pprstblcount))
# Find comparable staging table
for stgstbl in tbllstgs:
stgstblname = stgstbl[len(stagingPrefix):]
if stgstblname == pprstblname:
# Get staging count
stgstblpath = os.path.join(stagingPath, stgstbl)
stgstblcount = arcpy.GetCount_management(
stgstblpath).getOutput(0)
stgstblcountname = stgstblname
continue
# Report staging count status
if stgstblcount != dummyVal and stgstblcount != pprstblcount:
etgLib.log_info(
log,
'*****ERROR!!!***** preprod count = {0} but staging count = {1}'
.format(pprstblcount, stgstblcount))
elif stgstblcount == dummyVal:
etgLib.log_info(
log,
'{} not found in staging SDE'.format(pprstblname))
else:
etgLib.log_info(
log, '{0} - staging count = {1}'.format(
stgstblcountname, stgstblcount))
# Find comparable labels table
for lblgtbl in tblllblg:
if lblgtbl == pprstblname:
# Get labels count
lblgtblpath = os.path.join(labelsPath, lblgtbl)
lblgtblcount = arcpy.GetCount_management(
lblgtblpath).getOutput(0)
lblgtblcountname = lblgtbl
continue
# Report labels count status
if lblgtblcount != dummyVal and lblgtblcount != pprstblcount:
etgLib.log_info(
log,
'*****ERROR!!!***** preprod count = {0} but labels count = {1}'
.format(pprstblcount, lblgtblcount))
elif lblgtblcount == dummyVal:
etgLib.log_info(
log,
'{} not found in labels GDB'.format(pprstblname))
else:
etgLib.log_info(
log, '{0} - labels count = {1}'.format(
lblgtblcountname, lblgtblcount))
etgLib.log_process_time(log, starttime)
except Exception as e:
err_message = "ERROR while running {0}: {1}".format(script_name, e)
return err_message
def crs4_copy_tbls_stage(args):
# script name
script_name = os.path.basename(__file__)
# script parameters
sdePath = args[0]
gdbPath = args[1]
sdePrefix = args[2]
log = args[3]
# Set environment
arcpy.env.workspace = gdbPath
arcpy.env.overwriteOutput = True
arcpy.env.configkeyword = "GEOMETRY"
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
# variables
err_message = None
# excludeList = ["INSTRUMENT"]
excludeList = []
try:
if arcpy.Exists(gdbPath):
### Copy tables from local GDB to database
#*** NOTE: tables have been deleted from SDE previously via
#*** CRS2_emptyNDCSTGsde.py - but still check for existence
# List tables in GDB
tbll = arcpy.ListTables()
# Loop through the tables
etgLib.log_info(log, 'Copying tables to staging SDE...', True)
for tbl in tbll:
inTBLpath = os.path.join(gdbPath, tbl)
outTBLname = sdePrefix + tbl
outTBLpath = os.path.join(sdePath, outTBLname)
# Check whether table exists in SDE, if so - print warning
if arcpy.Exists(outTBLpath):
etgLib.log_info(
log,
'WARNING: {} exists in staging SDE'.format(outTBLname))
# Otherwise, copy
else:
# Ignore tables in exclude list
if tbl in excludeList:
etgLib.log_info(log, 'Ignoring {}'.format(tbl))
else:
# Copy table from GDB to SDE
arcpy.Copy_management(inTBLpath, outTBLpath, "Table")
# Count features and report number - warn if not equal
inCount = arcpy.GetCount_management(
inTBLpath).getOutput(0)
outCount = arcpy.GetCount_management(
outTBLpath).getOutput(0)
if inCount == outCount:
etgLib.log_info(
log, '{0} - Copied {1} entries to {2}'.format(
tbl, inCount, outTBLname))
else:
etgLib.log_info(
log,
'ERROR: {0} entries copied from {1} - {2} entries resultant in {3}'
.format(inCount, tbl, outCount, outTBLname))
else:
err_message = 'ERROR: GDB not found - {}'.format(gdbPath)
etgLib.log_process_time(log, starttime)
except Exception as e:
err_message = "ERROR while running {0}: {1}".format(script_name, e)
return err_message
def crs3_copy_fcs_stage(args):
# script name
script_name = os.path.basename(__file__)
# script parameters
sdePath = args[0]
gdbPath = args[1]
sdePrefix = args[2]
log = args[3]
# Set environment
arcpy.env.workspace = gdbPath
arcpy.env.overwriteOutput = True
arcpy.env.configkeyword = "GEOMETRY"
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
# variables
err_message = None
try:
if arcpy.Exists(gdbPath):
### Copy feature classes from local GDB to database
#*** NOTE: feature classes have been deleted from SDE previously via
#*** CRS2_emptyNDCSTGsde.py - but still check for existence
# List feature classes in GDB
fcl = arcpy.ListFeatureClasses()
# Loop through the FCs
etgLib.log_info(log, 'Copying feature classes to staging SDE...',
True)
for fc in fcl:
inFCpath = os.path.join(gdbPath, fc)
outFCname = sdePrefix + fc
outFCpath = os.path.join(sdePath, outFCname)
# Check whether FC exists in SDE, if so - print warning
if arcpy.Exists(outFCpath):
etgLib.log_info(
log,
'WARNING: {} exists in staging SDE'.format(outFCname))
# Otherwise, copy
else:
# Copy FC from GDB to SDE
arcpy.Copy_management(inFCpath, outFCpath, "FeatureClass")
# Count features and report number - warn if not equal
inCount = arcpy.GetCount_management(inFCpath).getOutput(0)
outCount = arcpy.GetCount_management(outFCpath).getOutput(
0)
if inCount == outCount:
etgLib.log_info(
log, '{0} - Copied {1} features to {2}'.format(
fc, inCount, outFCname))
else:
etgLib.log_info(
log,
'ERROR: {0} features copied from {1} - {2} features resultant in {3}'
.format(inCount, fc, outCount, outFCname))
else:
err_message = 'ERROR: GDB not found - {}'.format(gdbPath)
etgLib.log_process_time(log, starttime)
except Exception as e:
err_message = "ERROR while running {0}: {1}".format(script_name, e)
return err_message
def crs6_add_data_preparation(args):
# parameters
wkgFolder = args[0]
labelGDBname = args[1]
log = args[2]
# Set locations, etc
labelGDBpath = os.path.join(wkgFolder, labelGDBname)
fcEPrclPath = os.path.join(labelGDBpath, fcEsmntPrcl)
fcELeasePath = os.path.join(labelGDBpath, fcEsmntLease)
fcLPrclPath = os.path.join(labelGDBpath, fcLeasePrcl)
fcRCLPath = os.path.join(labelGDBpath, fcRoadCL)
fcRCdsslvPath = os.path.join(labelGDBpath, fcRCLdsslv)
fcCadP = fcCadastre + "_P"
fcCadPPath = os.path.join(labelGDBpath, fcCadP)
tblPLnkPath = os.path.join(labelGDBpath, tblPropLink)
fcPrclPLPath = os.path.join(labelGDBpath, fcPrclPLink)
tblPropPath = os.path.join(labelGDBpath, tblProperty)
fcCnnctPPath = os.path.join(labelGDBpath, fcConnectProp)
fcDsslvIDPath = os.path.join(labelGDBpath, dsslvIDFC)
# Set environment
arcpy.env.workspace = labelGDBpath
arcpy.env.overwriteOutput = True
arcpy.env.configkeyword = "GEOMETRY"
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
# variables
err_message = None
try:
### Easement lease
# Copy easement parcel
etgLib.log_info(log, 'Copying easement parcel data in labels gdb ...',
True)
arcpy.Copy_management(fcEPrclPath, fcELeasePath)
# Select records to append
etgLib.log_info(log, 'Selecting lease parcels to append...')
etgLib.delete_layer("leaseplyr")
arcpy.MakeFeatureLayer_management(fcLPrclPath, "leaseplyr")
parcelClause = '"PARCEL_INTENT" = ' + "'LCOV'" + ' OR "PARCEL_INTENT" = ' + "'EASM'"
arcpy.SelectLayerByAttribute_management("leaseplyr", "NEW_SELECTION",
parcelClause)
etgLib.log_info(log, 'Appending lease parcels...')
arcpy.Append_management("leaseplyr", fcELeasePath, "NO_TEST")
### Road CL
etgLib.log_info(log, 'Working on road data...', True)
if etgLib.field_exist(fcRoadCL, rfield) == False:
# Add field
arcpy.AddField_management(fcRoadCL, rfield, "TEXT", "", "",
rfieldlen)
# Calculate values
etgLib.log_info(log, 'Calculate values: {} ...'.format(rfield))
calcexpr = ('!{}!.upper() + ", " + !{}!.upper()').format(
statsfields[0][0], statsfields[1][0])
arcpy.CalculateField_management(fcRoadCL, rfield, calcexpr,
"PYTHON_9.3")
# Dissolve data, using statistics fields
etgLib.log_info(log, 'Dissolving ...')
arcpy.Dissolve_management(fcRCLPath, fcRCdsslvPath, rfield,
statsfields)
# Add fields
arcpy.AddField_management(fcRCLdsslv, statsfields[0][0], "TEXT", "",
"", sfieldlen)
arcpy.AddField_management(fcRCLdsslv, statsfields[1][0], "TEXT", "",
"", sfieldlen)
# Calculate values
sfields = []
for i in range(len(statsfields)):
sfields.append(statsfields[i][0])
arcpy.AddField_management(fcRCLdsslv, statsfields[i][0], "TEXT",
"", "", sfieldlen)
sfield = statsfields[i][1] + "_" + statsfields[i][0]
calcexpr = ('!{}!').format(sfield)
arcpy.CalculateField_management(fcRCLdsslv, statsfields[i][0],
calcexpr, "PYTHON_9.3")
### Connect_Property
etgLib.log_info(log, 'Working on Connect_Property', True)
# Make query table
mqtblList = [fcCadPPath, tblPLnkPath]
whereClause = tblPropLink + "." + parIDfield + " = " + fcCadP + "." + parIDfield # NOTE: no quotes - known bug
arcpy.MakeQueryTable_management(mqtblList, "propQueryTbl",
"ADD_VIRTUAL_KEY_FIELD", "", "",
whereClause)
# Get number of rows
numMQrows = int(arcpy.GetCount_management("propQueryTbl").getOutput(0))
# Export
etgLib.log_info(log, 'Exporting...')
arcpy.CopyFeatures_management("propQueryTbl", fcPrclPLPath)
# Check number of rows
numPPLrows = int(arcpy.GetCount_management(fcPrclPLPath).getOutput(0))
if numPPLrows != numMQrows:
etgLib.log_error(
log,
'ERROR: Wrong number of rows exported for link FC; {} versus {}'
.format(numMQrows, numPPLrows))
else:
etgLib.log_info(log,
'Correct number of rows exported for link FC.')
# Dissolve on ID
etgLib.log_info(log, 'Dissolving on ID...', True)
dfield = tblPropLink + "_" + propIDfield
sfield = tblPropLink + "_" + parIDfield
statsfield = [[sfield, "COUNT"]]
arcpy.Dissolve_management(fcPrclPLink, fcDsslvIDPath, dfield,
statsfield)
# Join the TP_Property table
etgLib.log_info(log, 'Preparing to join property table...')
# Create temporary layer/view
etgLib.delete_layer('dsslvlyr')
arcpy.MakeFeatureLayer_management(fcDsslvIDPath, "dsslvlyr")
etgLib.delete_layer('proptblview')
arcpy.MakeTableView_management(tblPropPath, "proptblview")
# Make join
etgLib.log_info(log, 'Adding join ...')
arcpy.AddJoin_management("dsslvlyr", dfield, "proptblview",
propIDfield, "KEEP_ALL")
etgLib.log_info(log, 'Property table joined')
# Output
etgLib.log_info(log, 'Copying features...')
arcpy.CopyFeatures_management("dsslvlyr", fcCnnctPPath)
etgLib.log_process_time(log, starttime)
except Exception as e:
err_message = "ERROR while running {0}: {1}".format(script_name, e)
return err_message
def crs5_prepare_for_labels(args):
wkgFolder = args[0]
labelGDBname = args[1]
sdePath = args[2]
dataSDEprefix = args[3]
log = args[4]
# Set locations, etc
labelGDBpath = os.path.join(wkgFolder, labelGDBname)
fcCdstrPath = os.path.join(labelGDBpath, fcCadastre)
fcCadP = fcCadastre + "_P"
fcCadPPath = os.path.join(labelGDBpath, fcCadP)
tblPLblPath = os.path.join(labelGDBpath, tblPrclLabel)
tblCLblPath = os.path.join(labelGDBpath, tblCdstrLabel)
# Set environment
arcpy.env.workspace = wkgFolder
arcpy.env.overwriteOutput = True
arcpy.env.configkeyword = "GEOMETRY"
# variables
err_message = None
try:
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
# start time
starttime = datetime.datetime.now()
### Create labels GDB - check for existence first
etgLib.log_info(log, 'Creating working labels GDB...')
if arcpy.Exists(labelGDBpath):
etgLib.log_info(log,
'WARNING: {} already exists!'.format(labelGDBpath))
else:
arcpy.CreateFileGDB_management(wkgFolder, labelGDBname)
### Copy feature classes from staging database (SDE) to local GDB
etgLib.log_info(
log,
'Copying feature classes from staging dabase to labels.gdb ...',
True)
for fc in fcsToCopy:
inFCname = dataSDEprefix + fc
inFCpath = os.path.join(sdePath, inFCname)
outFCpath = os.path.join(labelGDBpath, fc)
# Check whether FC exists in GDB, if so - overwrite
if arcpy.Exists(outFCpath):
etgLib.log_info(
log,
'WARNING: {} already exists - overwriting...'.format(fc))
# Check whether table exists in SDE, if so - continue
if arcpy.Exists(inFCpath):
arcpy.Copy_management(inFCpath, outFCpath)
# Count features and report number - warn if not equal
inCount = arcpy.GetCount_management(inFCpath).getOutput(0)
outCount = arcpy.GetCount_management(outFCpath).getOutput(0)
if inCount == outCount:
etgLib.log_info(
log, '{0} - Copied {1} features to {2}'.format(
inFCname, inCount, fc))
else:
etgLib.log_info(
log,
'ERROR: {0} features copied from {1} - {2} features resultant in {3}'
.format(inCount, inFCname, outCount, fc))
else:
err_message = '{} does not exist - exit...'.format(fc)
return err_message
### Copy tables from staging database (SDE) to local GDB
etgLib.log_info(
log, 'Copying tables from staging dabase to labels.gdb ...', True)
for tbl in tblsToCopy:
inTBLname = dataSDEprefix + tbl
inTBLpath = os.path.join(sdePath, inTBLname)
outTBLpath = os.path.join(labelGDBpath, tbl)
# Check whether table exists in GDB, if so - overwrite
if arcpy.Exists(outTBLpath):
etgLib.log_info(
log,
'WARNING: {} already exists - overwriting...'.format(tbl))
# Check whether table exists in SDE, if so - continue
if arcpy.Exists(inTBLpath):
arcpy.Copy_management(inTBLpath, outTBLpath)
# Count features and report number - warn if not equal
inCount = arcpy.GetCount_management(inTBLpath).getOutput(0)
outCount = arcpy.GetCount_management(outTBLpath).getOutput(0)
if inCount == outCount:
etgLib.log_info(
log, '{0} - Copied {1} entries to {2}'.format(
inTBLname, inCount, tbl))
else:
etgLib.log_info(
log,
'ERROR: {0} features copied from {1} - {2} features resultant in {3}'
.format(inCount, inTBLname, outCount, tbl))
else:
err_message = '{} does not exist - exit...'.format(tbl)
return err_message
### Work on cadastre dataset
etgLib.log_info(log, 'Adding fields to cadastre...', True)
# Change workspace location
arcpy.env.workspace = labelGDBpath
## Add fields for label coordinates
arcpy.AddField_management(fcCadastre, xlbl, "DOUBLE")
arcpy.AddField_management(fcCadastre, ylbl, "DOUBLE")
## Calculate x,y values
etgLib.log_info(log, 'calculating xlabel ylabel field values ...')
# Change workspace location
with arcpy.da.UpdateCursor(fcCdstrPath,
["OID@", "SHAPE@", xlbl, ylbl]) as cursor:
for row in cursor:
lPt = row[1].labelPoint
row[2] = lPt.X
row[3] = lPt.Y
cursor.updateRow(row)
# Delete cursor and row objects
del cursor, row
### Select "P" type parcels and export
etgLib.log_info(log, 'Select P type parcels and export ...', True)
# Check whether dataset exists already
if arcpy.Exists(fcCadPPath):
etgLib.log_info(
log, '"P" type parcel dataset already exists; overwriting...')
else:
etgLib.log_info(log, 'Exporting "P" type parcels...')
# Select "P" type parcels
etgLib.delete_layer("cadastrelyr")
arcpy.MakeFeatureLayer_management(fcCdstrPath, "cadastrelyr")
parcelClause = '"PARCEL_CATEGORY" = ' + "'P'"
arcpy.SelectLayerByAttribute_management("cadastrelyr", "NEW_SELECTION",
parcelClause)
# Export selected parcels
arcpy.CopyFeatures_management("cadastrelyr", fcCadPPath)
### Join "P" parcel data to label table and export
etgLib.log_info(log, 'Joining "P" type parcels to label table...')
etgLib.delete_layer("cadplyr")
arcpy.MakeFeatureLayer_management(fcCadPPath, "cadplyr")
arcpy.MakeTableView_management(tblPLblPath, "labelview")
arcpy.AddJoin_management("cadplyr", joinFieldP1, tblPLblPath,
joinFieldP2, "KEEP_COMMON")
####################### Block to set field names in parcel_label_pt correctly
arcpy.TableToTable_conversion("cadplyr", labelGDBpath, "junktable")
etgLib.log_info(log, 'Junk table created.')
etgLib.delete_layer("tmptbl")
arcpy.MakeTableView_management("junktable", "tmptbl")
etgLib.log_info(log, 'Describing temporary table...')
desc = arcpy.Describe("tmptbl")
fieldInfo = desc.fieldInfo
index = 0
etgLib.log_info(log, 'Updating field names...')
while index < fieldInfo.count:
for of1 in outfieldsP1:
of1longname = tblPrclLabel + "_" + of1
of1_1 = of1 + "_1"
if fieldInfo.getFieldName(index) == of1longname:
fieldInfo.setNewName(index, of1_1)
for of2 in outfieldsP2:
of2longname = tblPrclLabel + "_" + of2
if fieldInfo.getFieldName(index) == of2longname:
fieldInfo.setNewName(index, of2)
index += 1
etgLib.log_info(log, 'Field names converted.')
etgLib.delete_layer("tmptbl2")
arcpy.MakeTableView_management("tmptbl", "tmptbl2", "", "", fieldInfo)
etgLib.log_info(log, 'Made table view.')
arcpy.TableToTable_conversion("tmptbl2", labelGDBpath, tblCdstrLabel)
############################################################################################
etgLib.log_info(log, 'Exported table.', True)
arcpy.RemoveJoin_management("cadplyr")
### Create a point feature class based on the exported table
# Make a temporary event layer
arcpy.MakeXYEventLayer_management(tblCLblPath, xlbl, ylbl,
"XYeventlyr", spRef)
# Output the event layer to the point feature class
arcpy.FeatureClassToFeatureClass_conversion("XYeventlyr", labelGDBpath,
pLabelPt)
etgLib.log_process_time(log, starttime)
except Exception as e:
print("ERROR: {}".format(e))
err_message = "ERROR while running {0}: {1}".format(script_name, e)
return err_message
def crs7_remove_old_data_from_preprod(args):
# parameters
preprodPath = args[0]
preprodPrefix = args[1]
log = args[2]
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
# variables
err_message = None
try:
##### Find and delete any old data (*_o) from previous month's processing
# Get lists of tables and FCs
etgLib.log_info(log, 'Getting lists of tables and FCs from {} ...'.format(preprodPath))
arcpy.env.workspace = preprodPath
fcl = arcpy.ListFeatureClasses()
tbll = arcpy.ListTables()
# Work through list of feature classes
etgLib.log_info(log, '==> Checking preprod FCs for old data...',True)
for fc in fcl:
ppfcname = fc[len(preprodPrefix):]
# if fc.endswith("_o") or fc.endswith("_oo"):
if ppfcname.endswith("_o") or ppfcname.endswith("_oo"):
etgLib.log_info(log, 'Removing: {} ...'.format(fc))
if arcpy.Exists(fc):
arcpy.Delete_management(fc)
# Work through list of tables
etgLib.log_info(log, '==> Checking preprod tables for old data...',True)
for tbl in tbll:
pptblname = tbl[len(preprodPrefix):]
# if tbl.endswith("_o") or tbl.endswith("_oo"):
if pptblname.endswith("_o") or pptblname.endswith("_oo"):
etgLib.log_info(log, 'Removing: {} ...'.format(tbl))
if arcpy.Exists(tbl):
arcpy.Delete_management(tbl)
etgLib.log_process_time(log,starttime)
except Exception as e:
err_message = "ERROR while running {0}: {1}" .format(script_name,e)
return err_message
def crs2_empty_stage_sde(args):
# script name
script_name = os.path.basename(__file__)
# script parameters
sde = args[0]
log = args[1]
# Set environment
arcpy.env.workspace = sde
print(sde)
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
# variables
err_message = None
try:
# clear workspce cache
arcpy.ClearWorkspaceCache_management()
# Find all feature classes and delete a subset
fcl = arcpy.ListFeatureClasses()
etgLib.log_info(log, 'Deleting subset of feature classes:')
for fc in fcl:
print(fc)
if fc in itemsToKeep:
etgLib.log_info(log, 'Kept:{}'.format(fc))
else:
try:
arcpy.Delete_management(fc)
etgLib.log_info(log, 'Deleted: {}'.format(fc))
except:
# print('***ERROR*** while deleting {} - delete manually!!!').format(fc)
err_message = 'ERROR: deleting {}\n'.format(fc)
# Find all tables and delete a subset
tbll = arcpy.ListTables()
etgLib.log_info(log, 'Deleting subset of feature tables:')
for tbl in tbll:
if tbl in itemsToKeep:
etgLib.log_info(log, 'Kept:{}'.format(tbl))
else:
try:
arcpy.Delete_management(tbl)
etgLib.log_info(log, 'Deleted: {}'.format(tbl))
except:
if err_message != None:
err_message = err_message + 'ERROR: deleting {}\n'.format(
tbl)
else:
err_message = 'ERROR: deleting {}\n'.format(tbl)
etgLib.log_process_time(log, starttime)
except Exception as e:
err_message = "ERROR while running {0}: {1}".format(script_name, e)
return err_message
def crs1_repair_crs_data(args):
# script name
script_name = os.path.basename(__file__)
# script parameters
gdb = args[0]
log = args[1]
# Set environment
arcpy.env.workspace = gdb
# log function
etgLib.log_info(log, 'calling {}'.format(script_name), True)
# start time
starttime = datetime.datetime.now()
# variables
err_message = None
fc_cadastre = 'CADASTRE'
fld_cadastre = 'F_issues'
fc_ownerfixed = 'OWNER_FIXED'
fc_plan ='PLAN'
fc_parcelplan = 'PARCEL_PLAN'
try:
## Delete any extraneous data
etgLib.log_info(log, 'Delete any extraneous data',True)
etgLib.delete_layer(fc_ownerfixed)
# list all the feature classes
pointfcl = []
linefcl = []
polyfcl = []
tbll = []
fcs = arcpy.ListFeatureClasses()
for fc in fcs:
desc = arcpy.Describe(fc)
if desc.shapeType == 'Point':
pointfcl.append(fc)
elif desc.shapeType == 'Polyline':
linefcl.append(fc)
elif desc.shapeType == 'Polygon':
polyfcl.append(fc)
else:
pass
etgLib.log_info(log, 'Point feature classes:',True)
print_list(pointfcl,log)
etgLib.log_info(log, 'Polyline feature classes:',True)
print_list(linefcl, log)
etgLib.log_info(log, 'Polygon feature classes:',True)
print_list(polyfcl, log)
etgLib.log_info(log, 'tables:',True)
tbll = arcpy.ListTables()
print_list(tbll, log)
# add field to CADASTRE
if etgLib.field_exist(fc_cadastre, fld_cadastre) == False:
etgLib.log_info(log, 'adding field [{0}] in {1}'.format(fld_cadastre,fc_cadastre))
arcpy.AddField_management(fc_cadastre,fld_cadastre,"TEXT","","",250)
# Repair polygon geometry
etgLib.log_info(log,'Repairing polygon geometries...')
for polyfc in polyfcl:
preCount = arcpy.GetCount_management(polyfc).getOutput(0)
arcpy.RepairGeometry_management(polyfc)
postCount = arcpy.GetCount_management(polyfc).getOutput(0)
etgLib.log_info(log,'{0}: features pre-repair {1} - post-repair {2}'.format(polyfc, preCount, postCount))
# Rename PLAN to PARCEL_PLAN
if arcpy.Exists(fc_plan):
arcpy.Rename_management(fc_plan,fc_parcelplan)
etgLib.log_info(log,'Renamed {0} to {1}'.format(fc_plan,fc_parcelplan))
else:
etgLib.log_info(log,'ERROR: feature class {} not found'.format(fc_plan))
etgLib.log_process_time(log,starttime)
# ### Note that number of features pre- and post-repair should be emailed to Technical Specialist
# print('***NOTE: next step/s = email pre- and post-repair feature counts to Technical Specialist...')
except Exception as e:
err_message = "ERROR while running {0}: {1}" .format(script_name,e)
return err_message
def print_list(lst, log):
for itm in lst:
etgLib.log_info(log, itm)
def crs8_extract_for_connection(args):
wkgFolder = args[0]
assetsGDBname = args[1]
spreportSdePath = args[2]
spreportDataSDEprefix = args[3]
stgSdePath = args[4]
stgDataSDEprefix = args[5]
log = args[6]
# variables
err_message = None
# Set locations, etc
assetsGDBpath = os.path.join(wkgFolder, assetsGDBname)
try:
### Create assets GDB - check for existence first
etgLib.log_info(log, 'Creating working assets GDB...', True)
if arcpy.Exists(assetsGDBpath):
etgLib.log_info(
log,
'WARNING: {} already exists! - deleted'.format(assetsGDBpath))
arcpy.Delete_management(assetsGDBpath)
arcpy.CreateFileGDB_management(wkgFolder, assetsGDBname)
arcpy.env.workspace = assetsGDBpath
# Set the configKeyword environment to SDELOB.
#Process: Copy Features
etgLib.log_info(
log, 'Copy Features from SPREPORT database to assets gdb ...',
True)
for fc in fcsToCopy:
inFCname = spreportDataSDEprefix + fc
inFCpath = os.path.join(spreportSdePath, inFCname)
outFCpath = os.path.join(assetsGDBpath, fc)
etgLib.log_info(log, 'Copying: {}'.format(inFCpath))
if arcpy.Exists(inFCpath):
arcpy.Copy_management(inFCpath, outFCpath, "FeatureClass")
sectionClause = '"status" <> ' + "'FUTURE'" ##KC replaced previous line with this and three following
fc_name = spreportDataSDEprefix + "MX_SECTION"
in_fc = os.path.join(spreportSdePath, fc_name)
out_fc = os.path.join(assetsGDBpath, fc_name)
etgLib.delete_layer("sectionLyr")
arcpy.MakeFeatureLayer_management(in_fc, "sectionLyr", sectionClause)
etgLib.log_info(log, 'Copying: {}'.format(in_fc))
arcpy.CopyFeatures_management("sectionLyr", out_fc)
etgLib.log_info(log, 'Copying staging cadstre ...')
fc_name = stgDataSDEprefix + "CADASTRE"
in_fc = os.path.join(stgSdePath, fc_name)
out_fc = os.path.join(assetsGDBpath, fc_name)
arcpy.Copy_management(in_fc, out_fc, "FeatureClass")
# Process: Intersect
etgLib.log_info(log, 'Intersecting ...', True)
idx = 0
for fc in fcsToIntersect:
in_fcs = [fc, 'CADASTRE']
out_fc = outFcs[idx]
data_type = dataTypes[idx]
arcpy.Intersect_analysis(in_fcs, out_fc, "ALL", "#", data_type)
etgLib.log_info(log, 'Intersecting: {}'.format(in_fcs))
idx = idx + 1
#for each in list count and output
etgLib.log_info(log, 'checking feature count ...', True)
fcl = arcpy.ListFeatureClasses()
for idata in fcl:
#count records
fc_count = arcpy.GetCount_management(idata).getOutput(0)
etgLib.log_info(log, '{0} - {1} Records'.format(idata, fc_count))
etgLib.log_process_time(log, starttime)
except Exception as e:
err_message = "ERROR while running {0}: {1}".format(script_name, e)
return err_message