def exit_sys(log, txt, start, sendmail = False):
etgLib.log_error(log, txt)
etgLib.log_close(log, start)
if sendmail:
emailSubject = script_name + " - Failed"
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText, emailAttachments, smtpServer)
sys.exit()
def main_func():
start = datetime.datetime.now()
labelsPath = os.path.join(wkgFolder, labelGDBname)
err_msg = None
args = []
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "preprod sde path: {0}".format(preprodSdePath))
etgLib.log_info(log,
"preprod sde prefix: {0}".format(preprodSdePrefix))
etgLib.log_info(log, "staging sde path: {0}".format(stgSdePath))
etgLib.log_info(log, "staging sde prefix: {0}".format(stgSdePrefix))
## ========================================
## Process: check work folder age
## ========================================
err_msg = etgLib.check_folder_age(wkgFolder, cutoffage)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
## ========================================
## Process: call CRS9_checkPreprodCounts
## ========================================
args = [
labelsPath, preprodSdePath, preprodSdePrefix, stgSdePath,
stgSdePrefix, log
]
err_msg = crs9_check_preprod_feature_counts(args)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
except Exception as e:
err_msg = str(e)
etgLib.log_error(log, "error in {0}: {1}".format(script_name, e))
etgLib.log_close(log, start)
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText,
emailAttachments, smtpServer)
def main_func():
start = datetime.datetime.now()
err_msg = None
args = []
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(
log, "preprod SPREPORT SDE path: {0}".format(prepSPREPORTpath))
etgLib.log_info(
log, "production SPREPORT SDE path: {0}".format(prodSPREPORTpath))
etgLib.log_info(log,
"preprod SPOWN SDE path: {0}".format(prepSPOWNpath))
etgLib.log_info(log,
"production SPOWN SDE path: {0}".format(prodSPOWNpath))
## ==============================================================
## Process: call CRS10_prodSPREPORTandSPOWNrefresh_with_fieldMap
## ==============================================================
args = [
prepSPREPORTpath, prodSPREPORTpath, prepSPOWNpath, prodSPOWNpath
]
err_msg, log_msgs = crs10_prod_db_refresh(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
except Exception as e:
err_msg = str(e)
etgLib.log_error(log, "error in {0}: {1}".format(script_name, e))
etgLib.log_close(log, start)
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText,
emailAttachments, smtpServer)
def main_func():
start = datetime.datetime.now()
err_msg = None
args = []
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "preprod sde path: {0}".format(preprodSdePath))
etgLib.log_info(log, "sde prefix: {0}".format(preprodSdePrefix))
## ========================================
## Process: call CRS7_removeOldDataFromPreprod
## ========================================
args = [preprodSdePath, preprodSdePrefix]
err_msg, log_msgs = crs7_remove_old_data_from_preprod(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
except Exception as e:
err_msg = str(e)
etgLib.log_error(log, "error in {0}: {1}".format(script_name, e))
etgLib.log_close(log, start)
if sendMail:
if err_msg != None:
emailSubject = emailSubject1 + " - Failed"
else:
emailSubject = emailSubject1 + " - Successful"
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText,
emailAttachments, smtpServer)
def main_func():
start = datetime.datetime.now()
err_msg = None
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
lbl_gdb = os.path.join(wkgFolder, labelGDBname)
# set workspace
arcpy.env.workspace = lbl_gdb
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "working folder: {0}".format(wkgFolder))
etgLib.log_info(log, "prepSPOWNpath: {0}".format(prepSPOWNpath))
etgLib.log_info(log, "label gdb: {0}".format(lbl_gdb))
## ========================================
## Process: copy data from labels.gdb to SPOWN in pre-prod
## ========================================
etgLib.log_info(
log, 'copy Connect_Property from labels.gdb to SPOWN in pre-prod',
True)
# copy Connect_Property to *_o in SPOWN sde
etgLib.log_info(log, 'copy Connect_Property to *_o in SPOWN sde ...')
inFCpath = os.path.join(prepSPOWNpath, spown_property_connect)
if arcpy.Exists(inFCpath):
out_fc_name = spown_property_connect + "_o"
outFCpath = os.path.join(prepSPOWNpath, out_fc_name)
etgLib.log_info(
log, 'Copying: {0} to {1}'.format(spown_property_connect,
out_fc_name))
arcpy.Copy_management(inFCpath, outFCpath, "FeatureClass")
# truncate and append
etgLib.log_info(log, 'truncate Connect_Property in SPOWN sde ...')
arcpy.TruncateTable_management(inFCpath)
# build the FieldMappings
etgLib.log_info(log, 'build the FieldMappings ...')
list_fields_to_map = []
list_fields_to_map.append(('TP_PROPERTY_LINK_PROPERTY_ID',
'tempDsslvID_TP_PROPERTY_LINK_PROPERTY_ID'))
list_fields_to_map.append(
('PROPERTY_ID', 'tempDsslvID_TP_PROPERTY_LINK_PROPERTY_ID'))
list_fields_to_map.append(
('PARCEL_ID', 'tempDsslvID_TP_PROPERTY_LINK_PROPERTY_ID'))
list_fields_to_map.append(
('COUNT_PARCEL_ID',
'tempDsslvID_COUNT_TP_PROPERTY_LINK_PARCEL_ID'))
## # Old mappings for old staging dataset
## list_fields_to_map.append(('PROPERTY_ID','tempDsslvID_TP_PROPERTY_LINK_PROPERTY_ID'))
## list_fields_to_map.append(('LEGAL_DESCRIPTION','TP_PROPERTY_LEGAL_DESCRIPTION'))
## list_fields_to_map.append(('TITLE_NO', 'TP_PROPERTY_TITLE_NO'))
## list_fields_to_map.append(('SUBADDRESS_ID', 'TP_PROPERTY_SUBADDRESS_ID'))
## list_fields_to_map.append(('ADDRESS_SOURCE', 'TP_PROPERTY_ADDRESS_SOURCE'))
## list_fields_to_map.append(('COUNT_PARCEL_ID', 'tempDsslvID_COUNT_TP_PROPERTY_LINK_PARCEL_ID'))
print('Got to in fc name: {0}'.format(labels_property_connect))
inFCpath = os.path.join(lbl_gdb, labels_property_connect)
print('InFCPath: {0}'.format(inFCpath))
outFCpath = os.path.join(prepSPOWNpath, spown_property_connect)
print('OutFCPath: {0}'.format(outFCpath))
fieldmappings = etgLib.get_field_mapping(inFCpath, outFCpath,
list_fields_to_map)
# appending data from labels.gdb to SPOWN preprod sde
etgLib.log_info(
log, 'appending data from labels.gdb to SPOWN preprod sde ...')
arcpy.Append_management([inFCpath],
outFCpath,
"NO_TEST",
field_mapping=fieldmappings)
except Exception as e:
err_msg = "ERROR while running {0}: {1}".format(script_name, e)
etgLib.log_close(log, start)
print("Finished!!! Please check the result in ArcMap or ArcCatalog")
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText,
emailAttachments, smtpServer)
def main_func():
start=datetime.datetime.now()
# dateStamp = start.strftime("%Y%m%d")
prj = os.path.join(wkgFolder,prjName)
lbl = os.path.join(wkgFolder,labelGDBname)
err_msg = None
args = []
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "working folder: {0}".format(wkgFolder))
etgLib.log_info(log, "arcpro project: {0}".format(prj))
etgLib.log_info(log, "lbl gdb: {0}".format(lbl))
# ## =====================================
# ## Process: call CRS_updateDataSourceAprx
# ## =====================================
# etgLib.log_info(log, 'calling CRS_updateDataSourceAprx', True)
# args = [prj,lbl]
# err_msg = crs_update_datasource(args)
# if err_msg != None:
# exit_sys(log, err_msg, start,sendMail)
## =====================================
## Process: call CRS_exportLabelToAnnoAprx
## =====================================
etgLib.log_info(log, 'calling CRS_exportLabelToAnnoAprx', True)
args = [prj,lbl]
err_msg = crs_label_to_annotation(args)
if err_msg != None:
exit_sys(log, err_msg, start,sendMail)
except Exception as e:
err_msg = str(e)
etgLib.log_error(log,"error in {0}: {1}".format(script_name,e))
etgLib.log_close(log, start)
print ("Finished!!! Please check the result in ArcMap or ArcCatalog")
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText, emailAttachments, smtpServer)
def main_func():
start = datetime.datetime.now()
# dateStamp = start.strftime("%Y%m%d")
mxd = os.path.join(wkgFolder, mxdName)
lbl = os.path.join(wkgFolder, labelGDBname)
err_msg = None
args = []
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "working folder: {0}".format(wkgFolder))
etgLib.log_info(log, "mxd: {0}".format(mxd))
etgLib.log_info(log, "lbl gdb: {0}".format(lbl))
etgLib.log_info(log, "extent string: {0}".format(extentString))
etgLib.log_info(log, "sde path: {0}".format(stgSdePath))
## ========================================
## Process: check work folder age
## ========================================
err_msg = etgLib.check_folder_age(wkgFolder, cutoffage)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
# =============================
# Process: check data existance
# assume: mxd is in \CRS\MMMYYYY folder, create_Annotations.mxd CRS.GDB is in \CRS\MMMYYY folder
# =============================
etgLib.log_info(log, "Checking data existance", True)
args = [mxd]
for f in args:
if not arcpy.Exists(f):
err = "file does not exist: {}".format(f)
exit_sys(log, err, start, sendMail)
## ========================================
## Process: call CRS5_prepareForCRSLables
## ========================================
args = [wkgFolder, labelGDBname, stgSdePath, stgSdePrefix, log]
err_msg = crs5_prepare_for_labels(args)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
## ========================================
## Process: check if data is in nz boundary
## ========================================
args = [lbl, extentString, ['PARCEL_LABEL_PT'], log]
err_msg = crs_check_data_extent(args)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
## =====================================
## Process: call CRS_updateDataSourceMxd
## =====================================
args = [mxd, lbl, log]
err_msg = crs_update_datasource_mxd(args)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
except Exception as e:
err_msg = str(e)
etgLib.log_error(log, "error in CRS_Main2: {0}".format(e))
etgLib.log_close(log, start)
print("Finished!!! Please check the result in ArcMap or ArcCatalog")
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText,
emailAttachments, smtpServer)
def main_func():
start = datetime.datetime.now()
err_msg = None
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
# set workspace
arcpy.env.workspace = contextdSdePath
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
# etgLib.log_info(log, "working folder: {0}".format(wkgFolder))
etgLib.log_info(log,
"pre prod CONTEXTSdePath: {0}".format(contextdSdePath))
## ===============================================================
## Process: add attribute index to PARCEL_LABEL_PT_1 feature class
## ===============================================================
etgLib.log_info(
log,
'add attribute indexes to PARCEL_LABEL_PT_1 feature class in CONTEXT sde...',
True)
fc_name = contextSdePrefix + fc_parcel_label_pt
inFCpath = os.path.join(contextdSdePath, fc_name)
if arcpy.Exists(inFCpath) == False:
exit_sys(
log,
"{} does not exist in context sde database".format(fc_name),
start, sendMail)
indexes = arcpy.ListIndexes(
inFCpath) # ----> Lists both attribute and spatial indexes!!!
indexPrefix = None
etgLib.log_info(log, 'get the existing index prefix ...')
for index in indexes:
m = re.search(r'^R(\d+)_pk', index.name)
if m != None:
indexPrefix = m.group(1)
break
if indexPrefix == None:
exit_sys(log, "no index prefix found in the existing indexes",
start, sendMail)
etgLib.log_info(log, 'index prefix : {0}'.format(indexPrefix))
list_index_to_create = []
list_index_to_create.append(
("Owneridx", ["owner1", "owner2", "owner3",
"owner4"])) # field names are case sensitive!
list_index_to_create.append(("VestSrvidx", ["VESTING_SURVEY_ID"]))
list_index_to_create.append(("FullAppidx", ["FULL_APP"]))
list_index_to_create.append(("Parcelidx", ["PARCEL_ID"]))
list_index_to_create.append(
("Titleidx", ["title1", "title2", "title3", "title4"]))
for index_to_create in list_index_to_create:
newIndex = "R{0}_{1}".format(indexPrefix, index_to_create[0])
#print('InFCPath: {0}, Index name: {1}'.format(inFCpath,newIndex))
fields = index_to_create[1]
etgLib.log_info(log, 'creating index : {0}'.format(newIndex))
arcpy.AddIndex_management(inFCpath, fields, newIndex, "NON_UNIQUE",
"NON_ASCENDING")
## ===============================================================
## Process: add attribute index to ROAD_CL_Dissove_1 feature class
## ===============================================================
etgLib.log_info(
log, 'add attribute indexes to ROAD_CL_Dissove_1 feature class...',
True)
fc_name = contextSdePrefix + fc_road_cl_dissolve
inFCpath = os.path.join(contextdSdePath, fc_name)
if arcpy.Exists(inFCpath) == False:
exit_sys(
log,
"{} does not exist in context sde database".format(fc_name),
start, sendMail)
indexes = arcpy.ListIndexes(
inFCpath) # ----> Lists both attribute and spatial indexes!!!
indexPrefix = None
etgLib.log_info(log, 'get the existing index prefix ...')
for index in indexes:
m = re.search(r'^R(\d+)_pk', index.name)
if m != None:
indexPrefix = m.group(1)
break
if indexPrefix == None:
exit_sys(log, "no index prefix found in the existing indexes",
start, sendMail)
etgLib.log_info(log, 'index prefix : {0}'.format(indexPrefix))
list_index_to_create = []
list_index_to_create.append(("Locidx", ["RD_LOC"]))
list_index_to_create.append(("Rdnameidx", ["ROAD_NAME1"]))
list_index_to_create.append(("Locatonidx", ["LOCATION1"]))
for index_to_create in list_index_to_create:
newIndex = "R{0}_{1}".format(indexPrefix, index_to_create[0])
fields = index_to_create[1]
etgLib.log_info(log, 'creating index : {0}'.format(newIndex))
arcpy.AddIndex_management(inFCpath, fields, newIndex, "NON_UNIQUE",
"NON_ASCENDING")
except Exception as e:
err_msg = "ERROR while running {0}: {1}".format(script_name, e)
etgLib.log_close(log, start)
print("Finished!!! Please check the result in ArcMap or ArcCatalog")
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText,
emailAttachments, smtpServer)
def main_func():
start=datetime.datetime.now()
crs = os.path.join(wkgFolder,crsName)
mxd = os.path.join(wkgFolder, mxdName)
err_msg = None
args = []
global emailAttachments
try:
# generate log file
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "Can't create a log file", start)
emailAttachments = [logfile]
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "working folder: {0}".format(wkgFolder))
etgLib.log_info(log, "mxd name: {0}".format(mxd))
etgLib.log_info(log, "crs gdb: {0}".format(crs))
etgLib.log_info(log, "extent string: {0}".format(extentString))
etgLib.log_info(log, "staging sde path: {0}".format(stgSdePath))
## ========================================
## Process: check work folder age
## ========================================
err_msg = etgLib.check_folder_age(wkgFolder, cutoffage)
if err_msg != None:
exit_sys(log, err_msg, start,sendMail)
# =============================
# Process: check data existance
# check if check_data.aprx and CRS.GDB are in \CRS\MMMYYY folder
# =============================
etgLib.log_info(log, "Checking data existance",True)
args = [prj,crs]
for f in args:
if not arcpy.Exists(f):
err = "file does not exist: {}".format(f)
exit_sys(log, err, start, sendMail)
## ========================================
## Process: check if data is in nz boundary
## ========================================
args = [crs, extentString, []]
err_msg, log_msgs = crs_check_data_extent(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start,sendMail)
## =====================================
## Process: call CRS_updateDataSourceMxd
## =====================================
args = [mxd,crs,log]
err_msg = crs_update_datasource_mxd(args)
if err_msg != None:
exit_sys(log, err_msg, start,sendMail)
## ================================
## Process: call CRS1_repairCRSdata
## ================================
args = [crs,log]
err_msg, log_msgs = crs1_repair_crs_data(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start,sendMail)
## ===============================
## Process: call CRS2_emptyNDCSTAG
## ===============================
args = [stgSdePath]
err_msg, log_msgs = crs2_empty_stage_sde(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start,sendMail)
## ===================================
## Process: call CRS3_copyFCsToNZCSTAG
## ===================================
args = [stgSdePath,crs,stgSdePrefix]
err_msg, log_msgs = crs3_copy_fcs_stage(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
## ======================================
## Process: call CRS4_copyTablesToNZCSTAG
## ======================================
args = [stgSdePath,crs,stgSdePrefix]
err_msg, log_msgs = crs4_copy_tbls_stage(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
except Exception as e:
err_msg = str(e)
etgLib.log_error(log,"error in CRS_Main1: {0}".format(e))
etgLib.log_close(log, start)
print ("Finished!!! Please check the result in ArcMap or ArcCatalog")
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText, emailAttachments, smtpServer)
def main_func():
start=datetime.datetime.now()
err_msg = None
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "Staging sde path: {0}".format(stgSdePath))
etgLib.log_info(log, "Context sde path: {0}".format(contextSdePath))
## ========================================
## Process: copy data from NZC_STG to NZCONTEXT
## ========================================
etgLib.log_info(log,'copy data from NZC_STG to NZCONTEXT ...',True)
# Get lists of tables and FCs
etgLib.log_info(log, 'Getting lists of tables and FCs from NZC_STG ...')
arcpy.env.workspace = stgSdePath
fcl = arcpy.ListFeatureClasses()
tbll = arcpy.ListTables()
# Process: Copy Feature classes
etgLib.log_info(log,'Copy Features from NZC_STG database to NZCONTEXT pre-prod SDE ...',True)
for fc in fcl:
## inFCname = stgSdePrefix + fc
inFCname = fc
etgLib.log_info(log,'Featureclass name: {}'.format(fc))
inFCpath = os.path.join(stgSdePath,inFCname)
etgLib.log_info(log,'Featureclass path: {}'.format(inFCpath))
NZC_fc = contextSdePrefix + fc.split(".")[2]
## out_fc_name = contextSdePrefix + fc + "_1"
out_fc_name = NZC_fc + "_1"
etgLib.log_info(log,'Out Featureclass: {}'.format(out_fc_name))
etgLib.log_info(log,'Context SDE Path: {}'.format(contextSdePath))
outFCpath = os.path.join(contextSdePath,out_fc_name)
etgLib.log_info(log,'Out fc: {}'.format(outFCPath))
etgLib.log_info(log,'Copying ...: {}'.format(inFCname))
arcpy.Copy_management(inFCpath,outFCpath, "FeatureClass")
# Process: Copy Feature classes
etgLib.log_info(log,'Copy Tables from NZC_STG database to NZCONTEXT pre-prod SDE ...',True)
for tbl in tbll:
if tbl not in excludedTables:
inTBLname = stgSdePrefix + tbl
## inTBLpath = os.path.join(stgSdePrefix,inTBLname)
inTBLpath = os.path.join(stgSdePath,inTBLname)
out_tbl_name = contextSdePrefix + tbl + "_1"
outTBLpath = os.path.join(contextSdePath,out_tbl_name)
etgLib.log_info(log,'Copying ...: {}'.format(inTBLname))
arcpy.Copy_management(inTBLpath,outTBLpath)
# Copy TITLE in CONTEXT to TITLE_o
etgLib.log_info(log,'Copy TITLE table to TITLE_o in NZCONTEXT pre-prod SDE ...',True)
inTBLname = contextSdePrefix + "TITLE"
inTBLpath = os.path.join(contextSdePath,inTBLname)
out_tbl_name = contextSdePrefix + "TITLE_o"
outTBLpath = os.path.join(contextSdePath,out_tbl_name)
arcpy.Copy_management(inTBLpath,outTBLpath)
# truncate and append Title in CONTEXT
etgLib.log_info(log,'truncate and append TITLE in CONTEXT ...')
arcpy.TruncateTable_management(inTBLname)
stg_title_table = stgSdePrefix + "TITLE"
in_tables = [stg_title_table]
arcpy.Append_management(in_tables, inTBLpath, "TEST")
# copying annotation layers to CONTEXT sde
etgLib.log_info(log,'copying annotation layers from labels.gdb to CONTEXT sde ...',True)
annoLayers = ['PARCEL_LABELSAnno2_5k', 'PARCEL_LABELSAnno5k', 'PARCEL_LABELSAnno10k', 'PARCEL_LABELSAnno20k','PARCEL_LABEL_PT', 'EASEMENT_LEASE', 'ROAD_CL_Dissolve']
lbl_gdb = os.path.join(wkgFolder,labelGDBname)
arcpy.env.workspace = lbl_gdb
for fc in annoLayers:
inFCpath = os.path.join(lbl_gdb,fc)
out_fc_name = contextSdePrefix + fc
outFCpath = os.path.join(contextSdePath,out_fc_name)
if arcpy.Exists(inFCpath):
etgLib.log_info(log,'Copying ...: {}'.format(fc))
arcpy.Copy_management(inFCpath,outFCpath, "FeatureClass")
except Exception as e:
err_msg = "ERROR while running {0}: {1}" .format(script_name,e)
etgLib.log_close(log, start)
print ("Finished!!! Please check the result in ArcMap or ArcCatalog")
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText, emailAttachments, smtpServer)
def main_func():
start=datetime.datetime.now()
# dateStamp = start.strftime("%Y%m%d")
err_msg = None
args = []
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "working folder: {0}".format(wkgFolder))
## ========================================
## Process: call CRS6_addDataPreparation
## ========================================
args = [wkgFolder,labelGDBname]
err_msg, log_msgs = crs6_add_data_preparation(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start,sendMail)
## ========================================
## Process: call CRS7_removeOldDataFromPreprod
## ========================================
args = [contextdSdePath,contextSdePrefix]
err_msg, log_msgs = crs7_remove_old_data_from_preprod(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start,sendMail)
## ========================================
## Process: call CRS8_Extract_for_Connect
## ========================================
args = [wkgFolder, assetsGDBname, spreportSdePath,spreportSdePrefix,stgSdePath, stgSdePrefix]
err_msg, log_msgs = crs8_extract_for_connection(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start,sendMail)
except Exception as e:
err_msg = str(e)
etgLib.log_error(log,"error in CRS_Main3: {0}".format(e))
etgLib.log_close(log, start)
print ("Finished!!! Please check the result in ArcMap or ArcCatalog")
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText, emailAttachments, smtpServer)
def main_func():
start=datetime.datetime.now()
err_msg = None
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
assets_gdb = os.path.join(wkgFolder,assetsGDBname)
# set workspace
arcpy.env.workspace = assets_gdb
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "working folder: {0}".format(wkgFolder))
etgLib.log_info(log, "prepSREPORTpath: {0}".format(prepSPREPORTpath))
etgLib.log_info(log, "assets gdb: {0}".format(assets_gdb))
## ========================================
## Process: copy data from assets.gdb to SPREPORT in pre-prod
## ========================================
etgLib.log_info(log,'copy data from assets.gdb to SPREPORT in pre-prod...',True)
list_of_fields_to_map_List = []
# Parcel_site
list_of_fields_to_map = []
list_of_fields_to_map.append(('SITE','MXLOCATION'))
list_of_fields_to_map.append(('SITE_TYPE','type'))
list_of_fields_to_map.append(('STATUS', 'status'))
list_of_fields_to_map.append(('SITE_DESC', 'description'))
list_of_fields_to_map_List.append(list_of_fields_to_map)
# SPAN_Parcel
list_of_fields_to_map = []
list_of_fields_to_map.append(('TAG_NO','MXLOCATION'))
list_of_fields_to_map.append(('SITE','site'))
list_of_fields_to_map_List.append(list_of_fields_to_map)
# STRUCTURE_Parcel
list_of_fields_to_map = []
list_of_fields_to_map.append(('TAG_NO','MXLOCATION'))
list_of_fields_to_map.append(('TAG_TYPE','type'))
list_of_fields_to_map.append(('STATUS', 'status'))
list_of_fields_to_map.append(('STR_CAT_DESC', 'LongType'))
list_of_fields_to_map_List.append(list_of_fields_to_map)
# Section_Parcel
list_of_fields_to_map = []
list_of_fields_to_map_List.append(list_of_fields_to_map)
# backup sreport feature classes
etgLib.log_info(log,'backup sreport feature classes...')
for fc in spreportFClist:
inFCpath = os.path.join(prepSPREPORTpath,fc)
out_fc_name = fc + "_o"
outFCpath = os.path.join(prepSPREPORTpath,out_fc_name)
etgLib.log_info(log,'Copying: {0} to {1}'.format(fc,out_fc_name ))
arcpy.Copy_management(inFCpath,outFCpath, "FeatureClass")
arcpy.TruncateTable_management(inFCpath)
# append from assets.gdb to SPREPORT
i = 0
for fc in assets_fcs:
inFCpath = os.path.join(assets_gdb,fc)
out_fc_name = spreportFClist[i]
outFCpath = os.path.join(prepSPREPORTpath,out_fc_name)
list_fields_to_map = list_of_fields_to_map_List[i]
if len(list_fields_to_map) > 0:
fieldmappings = etgLib.get_field_mapping(inFCpath, outFCpath,list_fields_to_map)
etgLib.log_info(log,'appending data from : {0}'.format(inFCpath))
arcpy.Append_management(fc,outFCpath, "NO_TEST", fieldmappings)
else:
etgLib.log_info(log,'appending data from : {0}'.format(inFCpath))
arcpy.Append_management(inFCpath,outFCpath, "TEST")
i = i+1
# field calculation
etgLib.log_info(log,'field calculation...')
Input_FC = os.path.join(prepSPREPORTpath,"SPREPORT.MAPDATA.SPAN_Parcel")
arcpy.CalculateField_management(Input_FC, "EQN_TYPE", "'CON'", "PYTHON_9.3", "")
Input_FC = os.path.join(prepSPREPORTpath,"SPREPORT.MAPDATA.STRUCTURE_Parcel")
arcpy.CalculateField_management(Input_FC, "SITE", "!TAG_NO![:9]", "PYTHON_9.3", "")
except Exception as e:
err_msg = "ERROR while running {0}: {1}" .format(script_name,e)
etgLib.log_close(log, start)
print ("Finished!!! Please check the result in ArcMap or ArcCatalog")
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText, emailAttachments, smtpServer)
def main_func():
start = datetime.datetime.now()
# dateStamp = start.strftime("%Y%m%d")
#Change this depending on whether using MXD or APRX (ArcGIS Pro) project
prj = os.path.join(wkgFolder, prjName)
lbl = os.path.join(wkgFolder, labelGDBname)
err_msg = None
args = []
global emailAttachments
try:
log_path = os.path.join(sys.path[0], 'logs')
log, logfile = etgLib.create_log(log_path, log_name)
if log == None: exit_sys(log, "can't create a log file", start)
emailAttachments = [logfile]
etgLib.log_start(log)
etgLib.log_info(log, "script parameters:")
etgLib.log_info(log, "------------------------")
etgLib.log_info(log, "working folder: {0}".format(wkgFolder))
etgLib.log_info(log, "arcpro project: {0}".format(prj))
etgLib.log_info(log, "lbl gdb: {0}".format(lbl))
etgLib.log_info(log, "extent string: {0}".format(extentString))
etgLib.log_info(log, "sde path: {0}".format(stgSdePath))
# =============================
# Process: check data existance
# check if create_Annotations.mxd/.aprx is in \CRS\MMMYYY folder
# =============================
etgLib.log_info(log, "Checking data existance", True)
args = [prj]
for f in args:
if not arcpy.Exists(f):
err_msg = "file does not exist: {}".format(f)
exit_sys(log, err_msg, start, sendMail)
## ========================================
## Process: call CRS5_prepareForCRSLabels
## ========================================
args = [wkgFolder, labelGDBname, stgSdePath, stgSdePrefix]
err_msg, log_msgs = crs5_prepare_for_labels(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
## ========================================
## Process: check if data is in NZ boundary by calling CRS_checkDataExtent
## ========================================
args = [lbl, extentString, ['PARCEL_LABEL_PT']]
err_msg, log_msgs = crs_check_data_extent(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
## =====================================
## Process: call CRS_updateDataSourceAprx
## =====================================
etgLib.log_info(log, 'calling CRS_updateDataSourceAprx', True)
args = [prj, lbl]
err_msg, log_msgs = crs_update_datasource(args)
etgLib.log_info_all(log, log_msgs)
if err_msg != None:
exit_sys(log, err_msg, start, sendMail)
# ## =====================================
# ## Process: call CRS_exportLabelToAnnoAprx
# ## =====================================
# etgLib.log_info(log, 'calling CRS_exportLabelToAnnoAprx', True)
# args = [prj,lbl]
# err_msg, log_msgs = crs_label_to_annotation(args)
# etgLib.log_info_all(log, log_msgs)
# if err_msg != None:
# exit_sys(log, err_msg, start,sendMail)
except Exception as e:
err_msg = str(e)
etgLib.log_error(log, "error in CRS_Main2: {0}".format(e))
etgLib.log_close(log, start)
print("Finished!!! Please check the result in ARcGIS Pro or ArcCatalog")
if sendMail:
if err_msg != None:
emailSubject = 'Run {} - Failed'.format(script_name)
else:
emailSubject = 'Run {} - Successful'.format(script_name)
etgLib.send_email(emailFrom, emailTo, emailSubject, emailText,
emailAttachments, smtpServer)
