def delete_versions(userlist):
try:
log.write(datetime.datetime.now().strftime("%H:%M:%S %m/%d/%Y ") +
"Deleting versions...\n")
for version in arcpy.da.ListVersions(WORKSPACE_SA):
if (version.name in userlist):
arcpy.DeleteVersion_management(WORKSPACE_SA, version.name)
log.write(
datetime.datetime.now().strftime("%H:%M:%S %m/%d/%Y ") +
"Version {0}".format(version.name) + " deleted.\n")
except Exception:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
pymsg = datetime.datetime.now().strftime(
"%H:%M:%S %m/%d/%Y "
) + "An error has occured.\nTraceback info:\n" + tbinfo + "Error Info:\n" + " " + str(
sys.exc_info()[1])
if arcpy.GetMessages(2) != "":
msgs = "\n\n-ARCPY ERRORS-\n" + arcpy.GetMessages(
2) + "\n\nScript finished with errors."
else:
msgs = "\n\nScript finished with errors."
log.write(pymsg)
log.write(msgs)
log.write(datetime.datetime.now().strftime("%H:%M:%S %m/%d/%Y ") +
"Script failed. Recreating versions...\n")
recreate_versions()
log.write(datetime.datetime.now().strftime("%H:%M:%S %m/%d/%Y ") +
"Reaccepting connections...\n")
clean_up()
def deleteVersion(sde, versionName):
addMessage("Deleting Version...")
retcode = False
if versionName.split(".")[0] == versionName:
# default to DBO
versionName = "DBO." + versionName
if debug == True:
addMessage(sde + "|" + versionName)
try:
try:
retcode = arcpy.DeleteVersion_management(sde, versionName)
except:
retcode = arcpy.DeleteVersion_management(sde, versionName)
if debug == True:
addMessage(versionName + " version deleted")
retcode = True
except:
retcode = False
#addMessage("Error: " + str(sys.exc_value))
#addMessage(versionName + " version not deleted... Please delete manually in ArcCatalog")
#exit(-1)
return retcode
def deletever(key, workspace):
#loop keys for versions
#get version name and attemp to delete version
for k in key:
for key in k:
if 'version-name' in key:
try:
arcpy.DeleteVersion_management(
workspace,
k['version-name'],
)
print ' Delete version: %s' % k['version-name']
except:
logger.error("Delete version " + k['version-name'] +
" Failed.")
def append_to_esri_source(self, input_fc, esri_output_fc,
input_where_clause):
"""
Append to the esri source FC, including projecting to output coord system, creating a versioned FC of the
output in SDE, deleting from that versioned FC based on a where_clause, appending the new data, and then
posting that version
:param input_fc: the input source data
:param esri_output_fc: the output in SDE
:param input_where_clause: where clause used to add to/delete from the output esri FC
:return:
"""
logging.info(
'Starting vector_layer.append_to_esri_source for {0}'.format(
self.name))
fc_to_append = self.project_to_output_srs(input_fc, esri_output_fc)
logging.debug('Creating a versioned FL from esri_service_output')
arcpy.MakeFeatureLayer_management(esri_output_fc,
"esri_service_output_fl")
version_name = self.name + "_" + str(int(time.time()))
sde_workspace = os.path.dirname(esri_output_fc)
desc = arcpy.Describe(sde_workspace)
if hasattr(desc,
"datasetType") and desc.datasetType == 'FeatureDataset':
sde_workspace = os.path.dirname(sde_workspace)
del desc
if os.path.splitext(sde_workspace)[1] != '.sde':
logging.error('Could not find proper SDE workspace. Exiting.')
sys.exit(1)
arcpy.CreateVersion_management(sde_workspace, "sde.DEFAULT",
version_name, "PRIVATE")
arcpy.ChangeVersion_management("esri_service_output_fl",
'TRANSACTIONAL', 'gfw.' + version_name,
'')
if input_where_clause:
logging.debug(
'Deleting features from esri_service_output feature layer based on input_where_clause. '
'SQL statement: {0}'.format(input_where_clause))
arcpy.SelectLayerByAttribute_management("esri_service_output_fl",
"NEW_SELECTION",
input_where_clause)
# Delete the features selected by the input where_clause
arcpy.DeleteRows_management("esri_service_output_fl")
else:
logging.debug(
'No where clause for esri_service_output found; deleting all features before '
'appending from source')
# commented out at this already happens below with wc
# arcpy.MakeFeatureLayer_management("esri_service_output_fl", "fl_to_delete")
# arcpy.DeleteRows_management("fl_to_delete")
# arcpy.Delete_management("fl_to_delete")
sde_sql_conn = arcpy.ArcSDESQLExecute(sde_workspace)
esri_fc_name = os.path.basename(esri_output_fc) + '_evw'
# Find the min and max OID values
to_delete_oid_field = [
f.name for f in arcpy.ListFields(esri_output_fc)
if f.type == 'OID'
][0]
sql = 'SELECT min({0}), max({0}) from {1}'.format(
to_delete_oid_field, esri_fc_name)
to_delete_min_oid, to_delete_max_oid = sde_sql_conn.execute(sql)[0]
# If there are features to delete, do it
if to_delete_min_oid and to_delete_max_oid:
for wc in util.generate_where_clause(to_delete_min_oid,
to_delete_max_oid,
to_delete_oid_field,
1000):
logging.debug('Deleting features with {0}'.format(wc))
arcpy.MakeFeatureLayer_management("esri_service_output_fl",
"fl_to_delete", wc)
arcpy.DeleteRows_management("fl_to_delete")
arcpy.Delete_management("fl_to_delete")
else:
pass
esri_output_pre_append_count = int(
arcpy.GetCount_management("esri_service_output_fl").getOutput(0))
input_feature_count = int(
arcpy.GetCount_management(fc_to_append).getOutput(0))
logging.debug('Starting to append to esri_service_output')
# Find the min and max OID values
to_append_oid_field = [
f.name for f in arcpy.ListFields(fc_to_append) if f.type == 'OID'
][0]
to_append_min_oid, to_append_max_oid = cartodb.ogrinfo_min_max(
fc_to_append, to_append_oid_field)
for wc in util.generate_where_clause(to_append_min_oid,
to_append_max_oid,
to_append_oid_field, 1000):
logging.debug('Appending features with {0}'.format(wc))
arcpy.MakeFeatureLayer_management(fc_to_append, "fl_to_append", wc)
arcpy.Append_management("fl_to_append", "esri_service_output_fl",
"NO_TEST")
arcpy.Delete_management("fl_to_append")
logging.debug('Append finished, starting to reconcile versions')
arcpy.ReconcileVersions_management(
input_database=sde_workspace,
reconcile_mode="ALL_VERSIONS",
target_version="sde.DEFAULT",
edit_versions='gfw.' + version_name,
acquire_locks="LOCK_ACQUIRED",
abort_if_conflicts="NO_ABORT",
conflict_definition="BY_OBJECT",
conflict_resolution="FAVOR_TARGET_VERSION",
with_post="POST",
with_delete="KEEP_VERSION",
out_log="")
logging.debug('Deleting temporary FL and temporary version')
# For some reason need to run DeleteVersion_management here, will have errors if with_delete is used above
arcpy.Delete_management("esri_service_output_fl")
arcpy.DeleteVersion_management(sde_workspace, 'gfw.' + version_name)
post_append_count = int(
arcpy.GetCount_management(esri_output_fc).getOutput(0))
if esri_output_pre_append_count + input_feature_count == post_append_count:
logging.debug('Append successful based on sum of input features')
else:
logging.debug(
'esri_output_pre_append_count: {0}\input_feature_count: {1}\npost_append_count{2}\n'
'Append failed, sum of input features does not match. '
'Exiting'.format(esri_output_pre_append_count,
input_feature_count, post_append_count))
sys.exit(1)
return
arcpy.AddError(pymsg)
arcpy.AddError(msgs)
# Print Python error messages for use in Python / Python Window
print(pymsg)
print(msgs)
finally:
# If the temporary version or connection file were created, we need to
# delete them
if is_versioned_sde:
# Stop editing
if 'edit' in globals() and edit.isEditing:
edit.abortOperation()
edit.stopEditing(False)
arcpy.AddMessage("Stopped edit session")
# Delete temporary version from SDE
for existing in arcpy.ListVersions(sde_path):
if version in existing:
arcpy.DeleteVersion_management(sde_path, version)
arcpy.AddMessage("Deleted temporary SDE version: " + version)
# Delete temporary version connection file
if os.path.exists(connection):
os.remove(connection)
arcpy.AddMessage("Deleted connection file: " + connection)
- The versions that we were reconciling are ready to be deleted if they are not currently pointing to a version
- We are going to loop through the reconcile versions list and remove any versions that are still pointing to a replica
- The versions remaining in the reconcile list are ready to be cleaned (deleted) up because there are no maps/replicas pointing to them.
'''
# Using the list of versions associated with users/maps that we reconciled earlier. Remove any versions from the list that are still being used by a replica.
for replicaVer in replicaVersions:
if replicaVer in verReconcileList:
verReconcileList.remove(replicaVer)
# Loop through the versionsList and delete versions that are no longer being referenced by a replica.
# Since these versions are no longer being referenced by a replica we can assume it's safe to delete them.
if len(verReconcileList
) > 0: # check to see that the list is not empty
for version in verReconcileList:
try:
arcpy.DeleteVersion_management(TIworkspace, version)
except:
print("Failed to delete version.")
print(arcpy.GetMessages(2))
else:
print("No versions to delete.")
else:
print(
"No versions to reconcile, aborting version maintenance routine.")
except:
print(arcpy.GetMessages(2))
print("Thermal Inventory Reconciled")
def delete(self):
if self.exists():
arcpy.DeleteVersion_management(self.gdb.sdeconn
,self.name)
def delete_version(self, version_name): """Deletes the specified child version.""" arcpy.DeleteVersion_management(self.workspace, version_name)
LogFile = pathname + "\\log\\" + Date + " " + Time1 + "-GISCompress.txt"
output = open(LogFile, "w")
output.write("GISCompress\n")
output.write(strmsg1)
for fileLine in workspacelist:
ENV.workspace = fileLine
strmsg1 = "Compress Workspace: " + fileLine + "\n"
print strmsg1
output.write(strmsg1)
try:
versionList = arcpy.ListVersions(fileLine)
for version in versionList:
if version.find("{")>-1:
print "Delete Version: " + version
arcpy.DeleteVersion_management(fileLine, version)
print arcpy.GetMessages().encode("gb2312") + "\n"
output.write(arcpy.GetMessages().encode("gb2312")+ "\n")
arcpy.Compress_management(fileLine)
print arcpy.GetMessages().encode("gb2312") + "\n"
output.write(arcpy.GetMessages().encode("gb2312")+ "\n")
except:
print arcpy.GetMessages().encode("gb2312") + "\n"
output.write(arcpy.GetMessages().encode("gb2312")+ "\n")
Date = time.strftime("%m-%d-%Y", time.localtime())# Set the date.
Time = time.strftime("%I:%M:%S %p", time.localtime()) # Set the time.
output.write(str("Process completed at " + str(Date) + " " + str(Time) + "." + "\n")) # Write the start time to the log file.
output.close() # Closes the log file.
print "Process completed at " + str(Date) + " " + str(Time) + "."
def main():
env.overwriteOutput = True
arcpy.ClearWorkspaceCache_management()
# Staging SDE:
folderName = r"E:\gis\projects\Staging\Admin\prod_build"
fileName = [
"Lex_CDOT_Survey_Admin - Staging.sde",
"Lex_CDOT_MapOutput_Admin - Staging(DEFAULT).sde"
]
t_v = ["DBO.Survey", "dbo.DEFAULT"]
#Use connection files with SDE version for editing only
editSDE = r"E:\gis\projects\Staging\Admin\prod_build\Lex_CDOT_Preproc_Admin - Staging.sde"
editMapbookSDE = r"E:\gis\projects\Staging\Admin\prod_build\Lex_CDOT_MapOutput_Admin - Staging(PreProc).sde"
# Setup workspaces
preGDB = arcpy.GetParameterAsText(0)
if preGDB == "#" or not preGDB:
preGDB = r"E:\gis\projects\IL\Chicago\CDOT_Menu_Pro\Spatial\gdb\working\PreProcessing\Lex_CDOT_Menu_PreProcessing_01072015_125937.gdb"
preProjLines = preGDB + "\ProjectCenterline_draft"
preBlockLines = preGDB + "\BlockCenterline_draft"
desc = arcpy.Describe(preProjLines)
preGDB = desc.path
print "Path: " + desc.path
arcpy.AddMessage("Path: " + desc.path)
print "Base Name: " + desc.baseName
arcpy.AddMessage("Base Name: " + desc.baseName)
desc = arcpy.Describe(preBlockLines)
print "Base Name: " + desc.baseName
arcpy.AddMessage("Base Name: " + desc.baseName)
# Call for connection
f = fileName[0]
checkConn(f, folderName)
SDE_DEFAULT = folderName + '\\' + f
targetSDE = SDE_DEFAULT
descSDE = arcpy.Describe(targetSDE)
sdeName = descSDE.connectionProperties.database
print "sdeName = " + sdeName
try:
versionList = arcpy.ListVersions(targetMapbookSDE)
if 'DBO.PreProc' in versionList:
arcpy.DeleteVersion_management(
targetMapbookSDE, 'DBO.PreProc'
) #Cannot remove version and recreate since its used by the connection file.
arcpy.CreateVersion_management(targetMapbookSDE, t_v[1], "PreProc",
"PUBLIC")
else:
arcpy.CreateVersion_management(targetSDE, t_v[1], "PreProc",
"PUBLIC")
except Exception as e:
print "Problem creating version"
# Call for connection
f = fileName[1]
checkConn(f, folderName)
SDE_DEFAULT = folderName + '\\' + f
targetMapbookSDE = SDE_DEFAULT
descMB = arcpy.Describe(targetMapbookSDE)
mbSDEName = descMB.connectionProperties.database
print mbSDEName
preProjLinesCombo = preProjLines.replace("\ProjectCenterline",
"\ProjectCenterlineCombo")
preBlockLinesCombo = preBlockLines.replace("\BlockCenterline",
"\BlockCenterlineCombo")
env.workspace = preGDB
""" ************************************ Project Centerline Updates ****************************** """
print ""
print "***************************************"
print "Beginning Project Centerline Updates..."
print "***************************************"
# Create lists of needed field names
fieldsProject = [
"strCCEProjectNumber", "strCDOTProjectNumber", "memAldermanComment",
"memProjectComments", "memWardComment", "SHAPE@", "strReportLocation",
"strReportFrom", "strReportTo", "lngMenuID", "lngWardID", "lngBlocks",
"lngProjectID"
] #These are the needed fields for the ProjectCenterline
fieldsBlock = [
"PageNo", "strCDOTProjectNumber", "strBlockLocation", "strBlockFrom",
"strBlockTo", "SHAPE@", "strCCEProjectNumber", "BlockNo",
"lngProjectID"
]
allFields = "*"
#Add query to ignore RW and RC Project types
whereClause = """"strCCEProjectNumber" NOT LIKE '%RW%' AND "strCCEProjectNumber" NOT LIKE '%RC%'"""
#Start an editing session
edit = arcpy.da.Editor(editSDE)
edit.startEditing(False, True)
edit.startOperation()
#Create new SearchCursor and InsertCursor for ProjectCenterline_v3
s_curGDB = arcpy.da.SearchCursor(preProjLinesCombo, fieldsProject,
whereClause)
print "Search cursor set... ProjectCenterline. Evaluating Records..."
i_cursSDE = arcpy.da.InsertCursor(
editSDE + os.sep + sdeName + ".DBO.Ops\\" + sdeName +
".DBO.ProjectCenterline", fieldsProject)
print "Insert Cursor set... ProjectCenterline"
#Create a list to hold the records that do not have any geometry. This list will notify the user at the end of any records that have no geometry.
noGeometryProject = []
# Create a list to hold project numbers that have had blocks deleted from them.
projectsWithDeletedBlocks = []
# Start the for loop to cycle through all the rows
for s_row in s_curGDB:
shape = s_row[5] #This variable points to the Shape token, "SHAPE@"
match = False # A boolean value that is changed to True if the record matches a record already in the Target SDE.
# Setup update cursor for the Target SDE - This will cycle through the existing records in the SDE.
u_cursSDE = arcpy.da.UpdateCursor(
editSDE + os.sep + sdeName + ".DBO.Ops\\" + sdeName +
".DBO.ProjectCenterline", fieldsProject,
fieldsProject[0] + " = '" + s_row[0] + "'")
#If the feature has no geometry, it's PageNo is added to the noGeometryArray and reported to the user at the end of the script.
if shape == None:
noGeometryProject.append(str(s_row[0]))
#Cycle through the records in the Target SDE and compare the PageNo value with that in the GDB.
for u_row in u_cursSDE:
counter = 0
if u_row[0] == s_row[0]:
if u_row[1] == s_row[1]:
pass
else:
counter = counter + 1
u_row[1] = s_row[1]
if u_row[2] == s_row[2]:
pass
else:
counter = counter + 1
u_row[2] = s_row[2]
if u_row[3] == s_row[3]:
pass
else:
counter = counter + 1
u_row[3] = s_row[3]
if u_row[4] == s_row[4]:
pass
else:
counter = counter + 1
u_row[4] = s_row[4]
if u_row[6] == s_row[6]:
pass
else:
counter = counter + 1
u_row[6] = s_row[6]
if u_row[7] == s_row[7]:
pass
else:
counter = counter + 1
u_row[7] = s_row[7]
if u_row[8] == s_row[8]:
pass
else:
counter = counter + 1
u_row[8] = s_row[8]
if u_row[9] == s_row[9]:
pass
else:
counter = counter + 1
u_row[9] = s_row[9]
if u_row[10] == s_row[10]:
pass
else:
counter = counter + 1
u_row[10] = s_row[10]
if u_row[11] == s_row[11]:
pass
else:
# if value for lngBlocks is different, then a block has been deleted.
# Add project number to a list of projects with deleted blocks
if u_row[11] > s_row[11]:
projectsWithDeletedBlocks.append(s_row[0])
counter = counter + 1
u_row[11] = s_row[11]
if u_row[12] == s_row[12]:
pass
else:
counter = counter + 1
u_row[12] = s_row[12]
if counter > 0:
u_cursSDE.updateRow(u_row)
print "Update " + str(counter) + " objects to row."
print str(s_row[0]) + " was updated."
else:
print str(
s_row[0]) + ": All attributes match. Skipping Record."
pass #If a matching record is found, update the record
match = True #Boolean to determine if its a new project or not
if match == False: #If no matching record is found in the SDE, the Insert Cursor will insert the new record.
i_cursSDE.insertRow(s_row)
print str(s_row[0]) + " was inserted."
del u_cursSDE #Delete the Update Cursor before re-creating it in the next iteration of the loop.
del i_cursSDE, s_curGDB #Delete the InsertCursor pointing to the Target SDE and the Search Cursor pointing to the GDB.
print "Number of projects with deleted blocks = " + str(
len(projectsWithDeletedBlocks))
for proj in projectsWithDeletedBlocks:
print proj + "has a deleted block."
print "*************************************"
print "ProjectCenterline updating is complete."
print "*************************************"
arcpy.AddMessage("*************************************")
arcpy.AddMessage("ProjectCenterline updating is complete.")
arcpy.AddMessage("*************************************")
"""********************************* Block Centerline Updates **********************************"""
print ""
print "***************************************"
print "Beginning Block Centerline Updates..."
print "***************************************"
#Create Search Cursor - Cusor is initially set to the BlockCenterline_v3 dataset.
s_curGDB = arcpy.da.SearchCursor(preBlockLinesCombo, fieldsBlock,
whereClause)
print "Search cursor set... BlockCenterline. Evaluating Records..."
#Create Insert Cursor (It is more efficient if the Insert Cursor is created outside the loop)
i_cursSDE = arcpy.da.InsertCursor(
editSDE + os.sep + sdeName + ".DBO.Ops\\" + sdeName +
".DBO.BlockCenterline",
fieldsBlock) #Change for final once final SDE in place <SAH>
#Create a list to hold the records that do not have any geometry. This list will notify the user at the end of any records that have no geometry.
noGeometryBlock = []
# Set Update Cursor to point to the BlockCenterline layer in the SDE
u_cursSDE = arcpy.da.UpdateCursor(
editSDE + os.sep + sdeName + ".DBO.Ops\\" + sdeName +
".DBO.BlockCenterline", fieldsBlock)
# Cycle through the SDE's Block Centerline layer. All blocks with a project number found in the projectsWithDeletedBlocks list will be renamed to have an "XX" in place of their letters.
numBlocksDeleted = 0
for u_row in u_cursSDE:
if u_row[6] in projectsWithDeletedBlocks:
newRow = u_row # Create a copy of the current row
oldProjNum = u_row[6]
if len(
oldProjNum
) > 9: # Create new project number with XX in place of letters.
newProjNum = oldProjNum[0:3] + "YY" + oldProjNum[
5] + "YY" + oldProjNum[8:]
else:
newProjNum = oldProjNum[0:3] + "YY" + oldProjNum[5:]
newRow[6] = newProjNum # Give new row the modified project number.
newRow[0] = newProjNum + "_" + str(
u_row[7]) # Update PageNo with changed project number.
u_cursSDE.deleteRow() # Delete the old row.
i_cursSDE.insertRow(
newRow) # Insert the copy with the modified project number.
numBlocksDeleted = numBlocksDeleted + 1
print "Number of blocks deleted = " + str(numBlocksDeleted)
del u_cursSDE
# Start the for loop to cycle through all the rows
for s_row in s_curGDB:
shape = s_row[5] #This variable points to the Shape token, "SHAPE@"
match = False # A boolean value that is changed to True if the record matches a record already in the Target SDE.
# Setup update cursor for the Target SDE - This will cycle through the existing records in the SDE.
u_cursSDE = arcpy.da.UpdateCursor(
editSDE + os.sep + sdeName + ".DBO.Ops\\" + sdeName +
".DBO.BlockCenterline", fieldsBlock,
fieldsBlock[0] + " = '" + s_row[0] + "'")
#If the feature has no geometry, it's PageNo is added to the noGeometryArray and reported to the user at the end of the script.
if shape == None:
noGeometryBlock.append(str(s_row[0]))
#Cycle through the records in the Target SDE and compare the PageNo value with that in the GDB.
for u_row in u_cursSDE:
counter = 0
if u_row[0] == s_row[
0]: # If PageNo in preBlockLinesCombo == PageNo in the SDE
if u_row[1] == s_row[1]: #strCDOTProjectNumber
pass
else:
counter = counter + 1
u_row[1] = s_row[1]
if u_row[2] == s_row[2]: #strBlockLocation
pass
else:
counter = counter + 1
u_row[2] = s_row[2]
if u_row[3] == s_row[3]: #strBlockFrom
pass
else:
counter = counter + 1
u_row[3] = s_row[3]
if u_row[4] == s_row[4]: #strBlockTo
pass
else:
counter = counter + 1
u_row[4] = s_row[4]
if u_row[7] == s_row[7]: #BlockNo
pass
else:
counter = counter + 1
u_row[7] = s_row[7]
if u_row[8] == s_row[8]: #lngProjectID
pass
else:
counter = counter + 1
u_row[8] = s_row[8]
if counter > 0:
u_cursSDE.updateRow(u_row)
print "Update " + str(counter) + " objects to row."
print str(s_row[0]) + " was updated."
else:
print str(
s_row[0]) + ": All attributes match. Skipping Record."
pass #If a matching record is found, update the record
match = True
if match == False: #If no matching record is found in the SDE, the Insert Cursor will insert the new record.
i_cursSDE.insertRow(s_row)
print str(s_row[0]) + " was inserted."
del u_cursSDE #Delete the Update Cursor before re-creating it in the next iteration of the loop.
# ========================================================================================================================
del i_cursSDE, s_curGDB #Delete the InsertCursor pointing to the Target SDE and the Search Cursor pointing to the GDB.
#Stop the editing session.
edit.stopOperation()
edit.stopEditing(True)
del edit
# Posting to SDE
SDE_DEFAULT = targetSDE
version = "PreProc"
p = t_v[0]
Post(SDE_DEFAULT, version, p)
print "*************************************"
print "BlockCenterline updating is complete."
print "*************************************"
arcpy.AddMessage("*************************************")
arcpy.AddMessage("BlockCenterline updating is complete.")
arcpy.AddMessage("*************************************")
#-----------------------------------------------------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------------------------------------------------
"""Added Rotation to fieldsBlock List"""
fieldsBlock.append("Rotation")
arcpy.AddMessage(fieldsBlock)
#Start an editing session
edit1 = arcpy.da.Editor(editMapbookSDE)
edit1.startEditing(False, True)
edit1.startOperation()
#Create Search Cursor - Cusor is initially set to the BlockCenterline_v3 dataset.
s_curGDB = arcpy.da.SearchCursor(preBlockLines, fieldsBlock, whereClause)
print "Search cursor set... BlockCenterline. Evaluating Records..."
#Create Insert Cursor (It is more efficient if the Insert Cursor is created outside the loop)
i_cursSDE = arcpy.da.InsertCursor(
editMapbookSDE + os.sep + mbSDEName + ".DBO.Calcs\\" + mbSDEName +
".DBO.Blocks_FinalCalcs", fieldsBlock)
#Create a list to hold the records that do not have any geometry. This list will notify the user at the end of any records that have no geometry.
noGeometryBlock = []
# Start the for loop to cycle through all the rows
for s_row in s_curGDB:
shape = s_row[5] #This variable points to the Shape token, "SHAPE@"
match = False # A boolean value that is changed to True if the record matches a record already in the Target SDE.
# Setup update cursor for the Target SDE - This will cycle through the existing records in the SDE.
u_cursSDE = arcpy.da.UpdateCursor(
editMapbookSDE + os.sep + mbSDEName + ".DBO.Calcs\\" + mbSDEName +
".DBO.Blocks_FinalCalcs", fieldsBlock,
fieldsBlock[0] + " = '" + s_row[0] + "'")
#If the feature has no geometry, it's PageNo is added to the noGeometryArray and reported to the user at the end of the script.
if shape == None:
noGeometryBlock.append(str(s_row[0]))
#Cycle through the records in the Target SDE and compare the PageNo value with that in the GDB.
for u_row in u_cursSDE:
counter = 0
if u_row[0] == s_row[0]:
if u_row[1] == s_row[1]:
pass
else:
counter = counter + 1
u_row[1] = s_row[1]
if u_row[2] == s_row[2]:
pass
else:
counter = counter + 1
u_row[2] = s_row[2]
if u_row[3] == s_row[3]:
pass
else:
counter = counter + 1
u_row[3] = s_row[3]
if u_row[4] == s_row[4]:
pass
else:
counter = counter + 1
u_row[4] = s_row[4]
if u_row[7] == s_row[7]:
pass
else:
counter = counter + 1
u_row[7] = s_row[7]
if u_row[8] == s_row[8]:
pass
else:
counter = counter + 1
u_row[8] == s_row[8]
if counter > 0:
u_cursSDE.updateRow(
u_row
) #If a matching record is found, update the record
print "Update " + str(counter) + " objects to row."
print str(s_row[0]) + " was updated."
else:
print str(
s_row[0]) + ": All attributes match. Skipping Record."
pass
match = True #Boolean to determine if its a new project or not
if match == False: #If no matching record is found in the SDE, the Insert Cursor will insert the new record.
i_cursSDE.insertRow(s_row)
print str(s_row[0]) + " was inserted."
del u_cursSDE #Delete the Update Cursor before re-creating it in the next iteration of the loop.
del i_cursSDE, s_curGDB #Delete the InsertCursor pointing to the Target SDE and the Search Cursor pointing to the GDB.
print "*************************************"
print "MapBook BlockCenterline updating is complete."
print "*************************************"
arcpy.AddMessage("*************************************")
arcpy.AddMessage("MapBook BlockCenterline updating is complete.")
arcpy.AddMessage("*************************************")
#Stop the editing session.
edit1.stopOperation()
edit1.stopEditing(True)
del edit1
## Posting to SDE
SDE_DEFAULT = targetMapbookSDE
version = "PreProc"
p = t_v[1]
Post(SDE_DEFAULT, version, p)
#------------------------------------------------------------------------
#Notify the user of any features that do not have geometry.
if len(noGeometryBlock) > 0:
print ""
print "********************************************************** "
print "WARNING! The following Block Centerlines have no geometry and were skipped: "
arcpy.AddMessage(
"WARNING! The following Block Centerlines have no geometry and were skipped: "
)
for i in range(len(noGeometryBlock)):
print noGeometryBlock[i]
arcpy.AddMessage(noGeometryBlock[i])
print "********************************************************** "
elif len(noGeometryProject) > 0:
print ""
print "********************************************************** "
print "WARNING! The following Project Centerlines have no geometry and were skipped: "
arcpy.AddMessage(
"WARNING! The following Project Centerlines have no geometry and were skipped: "
)
for i in range(len(noGeometryProject)):
print noGeometryProject[i]
arcpy.AddMessage(noGeometryProject[i])
print "********************************************************** "
else:
print ""
print "*********************** "
print "The script is finished. "
print "*********************** "
arcpy.AddMessage("The script is finished. ")
arcpy.ClearWorkspaceCache_management()
from arcpy import env
env.overwriteOutput = True
d = (time.strftime("%Y_%m_%d"))
t1 = (time.strftime("%I:%M:%S"))
#Variables de entrada
#versionName=arcpy.GetParameterAsText(0)
# crear version
#print "Creando version: "+d+" "+t1
arcpy.AddMessage(" ")
#arcpy.AddMessage(u" Eliminando Versión "+versionName+" "+str(d)+" "+str(t1))
versionName = arcpy.GetParameterAsText(0)
idioma = arcpy.GetParameter(1)
if idioma == True:
inWorkspace = r'Conexiones de base de datos\SIRI_PRODUCCION.sde'
else:
inWorkspace = r'Database Connections\SIRI_PRODUCCION.sde'
arcpy.DeleteVersion_management(inWorkspace, versionName)
t2 = (time.strftime("%I:%M:%S"))
arcpy.AddMessage(" ")
arcpy.AddMessage(" Version Eliminada " + str(d) + " " + str(t2))
#print "Version eliminada"
def append_to_esri_source(self, input_fc, esri_output_fc,
input_where_clause):
"""
Append to the esri source FC, including projecting to output coord system, creating a versioned FC of the
output in SDE, deleting from that versioned FC based on a where_clause, appending the new data, and then
posting that version
:param input_fc: the input source data
:param esri_output_fc: the output in SDE
:param input_where_clause: where clause used to add to/delete from the output esri FC
:return:
"""
logging.info(
'Starting vector_layer.append_to_esri_source for {0}'.format(
self.name))
fc_to_append = self.project_to_output_srs(input_fc, esri_output_fc)
logging.info('appending {} to {}'.format(input_fc, esri_output_fc))
logging.debug('Creating a versioned FL from esri_service_output')
arcpy.MakeFeatureLayer_management(esri_output_fc,
"esri_service_output_fl")
version_name = self.name + "_" + str(int(time.time()))
sde_workspace = os.path.dirname(esri_output_fc)
desc = arcpy.Describe(sde_workspace)
if hasattr(desc,
"datasetType") and desc.datasetType == 'FeatureDataset':
sde_workspace = os.path.dirname(sde_workspace)
del desc
if os.path.splitext(sde_workspace)[1] != '.sde':
logging.error('Could not find proper SDE workspace. Exiting.')
sys.exit(1)
arcpy.CreateVersion_management(sde_workspace, "sde.DEFAULT",
version_name, "PRIVATE")
arcpy.ChangeVersion_management("esri_service_output_fl",
'TRANSACTIONAL', 'gfw.' + version_name,
'')
if input_where_clause:
logging.debug(
'Deleting features from esri_service_output feature layer based on input_where_clause. '
'SQL statement: {0}'.format(input_where_clause))
arcpy.SelectLayerByAttribute_management("esri_service_output_fl",
"NEW_SELECTION",
input_where_clause)
# Delete the features selected by the input where_clause
arcpy.DeleteRows_management("esri_service_output_fl")
else:
logging.debug(
'No where clause for esri_service_output found; deleting all features before '
'appending from source')
sde_sql_conn = arcpy.ArcSDESQLExecute(sde_workspace)
esri_fc_name = os.path.basename(esri_output_fc)
print esri_fc_name
# why this, exactly?
# there's also a lbr_plantations_old feature class (for some reason)
# and lbr_plantation_evw points to that.
# I don't know why and I don't have time to fix it
# WDPA test - leave off
if esri_fc_name != 'gfw_countries.gfw.lbr_plantations':
esri_fc_name += '_evw'
# Find the min and max OID values
to_delete_oid_field = [
f.name for f in arcpy.ListFields(esri_output_fc)
if f.type == 'OID'
][0]
sql = 'SELECT min({0}), max({0}) from {1}'.format(
to_delete_oid_field, esri_fc_name)
to_delete_min_oid, to_delete_max_oid = sde_sql_conn.execute(sql)[0]
# If there are features to delete, do it
if to_delete_min_oid and to_delete_max_oid:
for wc in util.generate_where_clause(to_delete_min_oid,
to_delete_max_oid,
to_delete_oid_field,
1000):
logging.debug('Deleting features with {0}'.format(wc))
arcpy.MakeFeatureLayer_management("esri_service_output_fl",
"fl_to_delete", wc)
arcpy.DeleteRows_management("fl_to_delete")
arcpy.Delete_management("fl_to_delete")
else:
pass
esri_output_pre_append_count = int(
arcpy.GetCount_management("esri_service_output_fl").getOutput(0))
input_feature_count = int(
arcpy.GetCount_management(fc_to_append).getOutput(0))
logging.debug('Starting to append to esri_service_output')
# don't need to batch append if it's coming from an SDE data source
# these are used exclusively by country-vector layers
# and the data is generally small, compared to things like WDPA
if 'sde' in fc_to_append:
logging.debug(
"Appending all features from {}- no wc because it's an SDE input"
.format(fc_to_append))
arcpy.MakeFeatureLayer_management(fc_to_append, "fl_to_append")
arcpy.Append_management("fl_to_append", "esri_service_output_fl",
"NO_TEST")
arcpy.Delete_management("fl_to_append")
else:
# Find the min and max OID values
to_append_oid_field = [
f.name for f in arcpy.ListFields(fc_to_append)
if f.type == 'OID'
][0]
to_append_min_oid, to_append_max_oid = cartodb.ogrinfo_min_max(
fc_to_append, to_append_oid_field)
for wc in util.generate_where_clause(to_append_min_oid,
to_append_max_oid,
to_append_oid_field, 1000):
logging.debug('Appending features with {0}'.format(wc))
arcpy.MakeFeatureLayer_management(fc_to_append, "fl_to_append",
wc)
arcpy.Append_management("fl_to_append",
"esri_service_output_fl", "NO_TEST")
arcpy.Delete_management("fl_to_append")
logging.debug('Append finished, starting to reconcile versions')
arcpy.ReconcileVersions_management(
input_database=sde_workspace,
reconcile_mode="ALL_VERSIONS",
target_version="sde.DEFAULT",
edit_versions='gfw.' + version_name,
acquire_locks="LOCK_ACQUIRED",
abort_if_conflicts="NO_ABORT",
conflict_definition="BY_OBJECT",
conflict_resolution="FAVOR_TARGET_VERSION",
with_post="POST",
with_delete="KEEP_VERSION",
out_log="")
logging.debug('Deleting temporary FL and temporary version')
# For some reason need to run DeleteVersion_management here, will have errors if with_delete is used above
arcpy.Delete_management("esri_service_output_fl")
arcpy.DeleteVersion_management(sde_workspace, 'gfw.' + version_name)
post_append_count = int(
arcpy.GetCount_management(esri_output_fc).getOutput(0))
if esri_output_pre_append_count + input_feature_count == post_append_count:
logging.debug('Append successful based on sum of input features')
else:
logging.debug(
'esri_output_pre_append_count: {0}\input_feature_count: {1}\npost_append_count{2}\n'
'Append failed, sum of input features does not match. '
'Exiting'.format(esri_output_pre_append_count,
input_feature_count, post_append_count))
sys.exit(1)
# now that we've finished syncing everything locally, need to push it to the PROD replica
local_sde = r'D:\scripts\connections\gfw (gfw@localhost).sde'
prod_gdb = r'P:\data\gfw_database\gfw.gdb'
arcpy.SynchronizeChanges_management(local_sde, "gfw.GFW", prod_gdb,
"FROM_GEODATABASE1_TO_2")
return
"POST", "KEEP_VERSION", reconcileLog)
# Write lines of "reconcileLog" to "log"
with open(reconcileLog, "r") as recFile: # Open rec file
for line in recFile: # Iterate lines in rec file
log.write(str(time.asctime()) + ": " +
line) # Write line to log file
recFile.close() # Close rec file
# Compress gdb
arcpy.Compress_management(defaultWorkspace)
log.write(str(time.asctime()) + ": Geodatabase compressed.\n")
# Delete versions
for version in versionList:
arcpy.DeleteVersion_management(defaultWorkspace, version)
log.write(str(time.asctime()) + ": " + version + " version deleted.\n")
# Compress gdb 2nd time
arcpy.Compress_management(defaultWorkspace)
log.write(
str(time.asctime()) + ": Geodatabase compressed for second time.\n")
# Allow the database to begin accepting connections again
arcpy.AcceptConnections(defaultWorkspace, True)
log.write(str(time.asctime()) + ": Database block removed.\n")
# Rebuild versions
connections = [sde, giseditor, sewerman]
versions = ["QC", "edits", "edits"]
pVersion = ["sde.Default", "SDE.QC", "SDE.QC"]
