def assign_fc_gisids(workspace, fc):
running_total = 0
cursor_counter = 0
batch_size = 100000
run_again = True
while run_again:
gisid = MC.read_gisid_seed()
edit = arcpy.da.Editor(workspace)
edit.startEditing(False, True)
edit.startOperation()
run_again = False
# UpdateCursor (in_table, field_names, {where_clause}, {spatial_reference}, {explode_to_points}, {sql_clause})
with arcpy.da.UpdateCursor(fc, ["OID@", "GISID"], """"GISID" IS NULL OR "GISID" = 0""") as cursor:
for row in cursor:
if cursor_counter > batch_size:
cursor_counter = 0
run_again = True
break
else:
row[1] = gisid
cursor.updateRow(row)
cursor_counter += 1
running_total += 1
gisid += 1
if cursor_counter > 0 and cursor_counter % 10000 == 0:
print cursor_counter
MC.write_gisid_seed(gisid)
edit.stopOperation()
edit.stopEditing(True)
logger.info(fc + "|" + str(running_total))
def clear_workspace_gisids(env):
for fc in MC.get_gps_fcs(env, True):
print fc
MC.disable_editor_tracking(env, fc)
arcpy.env.workspace = env
clear_fc_gisids(env, fc)
MC.enable_editor_tracking(env, fc)
def count_where_gisid_assigned(logname):
logger = MC.setup_logging(logname)
logger.info("Starting program...")
pp_workspace = MC.get_preprod_workspace()
p_workspace = MC.get_prod_workspace()
pp_dict = CC.count_fcs(pp_workspace, CC.count_fc_gisid_assigned)
p_dict = CC.count_fcs(p_workspace, CC.count_fc_gisid_assigned)
CC.write_r2d1_results(logname, logger, pp_dict, p_dict)
logger.info("Finished program")
def move_job_log(move_logger, source_workspace, target_workspace, state_of_interest):
fc = "NISOURCE.GPSFMEJobLog"
source_fc = os.path.join(source_workspace, fc)
target_fc = os.path.join(target_workspace, fc)
arcpy.env.workspace = target_workspace
dsc = arcpy.Describe(source_fc)
fields = dsc.fields
fieldnames = [field.name for field in fields if
field.name != dsc.OIDFieldName and field.name != 'GLOBALID' and field.name != 'SHAPE']
fieldnames_oid = fieldnames + ["OID@"]
state_index = fieldnames.index("TCCORGNAME")
sql_clause = (None, 'ORDER BY "OBJECTID"')
running_total = 0
with arcpy.da.SearchCursor(source_fc, fieldnames_oid, "", None, False, sql_clause) as sCur:
with arcpy.da.InsertCursor(target_fc, fieldnames) as iCur:
for row in sCur:
insert_row = mc.tuple_without(row, -1)
if (state_of_interest == 'OH' and sCur[state_index].lower() == 'nisourcecoh') or (
state_of_interest == 'PA' and sCur[state_index].lower() == 'nisourcefre') or (
state_of_interest == 'VA' and sCur[state_index].lower() == 'nisourcecgv'):
inserted_oid = iCur.insertRow(insert_row)
# print 'Match:', state_of_interest, sCur[-1]
running_total += 1
move_logger.info("Total|" + str(running_total))
def count_r2d2_wap(state_of_interest):
p_workspace = MC.get_r2d2_prod_workspace(state_of_interest)
arcpy.env.workspace = p_workspace
p_wap = ['NISOURCE.GPSWorkAreaPolygon']
p_dict = CC.count_list_fcs(p_wap, C4.count_r2d2_fc_state)
return p_dict
def delete_fcs(workspace_local):
arcpy.env.workspace = workspace_local
print workspace_local
for fc in mc.get_gps_fcs(workspace_local, True):
print fc
if fc != 'NISOURCE.GPS_CustomerConnection_G':
delete_fc(workspace_local, fc)
def move_workarea(move_logger, source_workspace, target_workspace, state_of_interest):
fc = "NISOURCE.GPSWorkAreaPolygon"
source_fc = os.path.join(source_workspace, fc)
target_fc = os.path.join(target_workspace, fc)
# MC.disable_editor_tracking(target_workspace, fc)
arcpy.env.workspace = target_workspace
dsc = arcpy.Describe(source_fc)
fields = dsc.fields
fieldnames = [field.name for field in fields if
field.name != dsc.OIDFieldName and field.name != 'GLOBALID' and field.name != 'SHAPE']
fieldnames = fieldnames + ["SHAPE@"]
fieldnames_oid = fieldnames + ["OID@"]
fieldnames_insert = fieldnames + ["GISID"]
state_index = fieldnames.index("WORKAREASTATE")
sql_clause = (None, 'ORDER BY "OBJECTID"')
running_total = 0
gisid = mc.read_gisid_seed()
edit = arcpy.da.Editor(target_workspace)
edit.startEditing(False, True)
edit.startOperation()
with arcpy.da.SearchCursor(source_fc, fieldnames_oid, "", None, False, sql_clause) as sCur:
with arcpy.da.InsertCursor(target_fc, fieldnames_insert) as iCur:
for row in sCur:
insert_row = mc.tuple_without(row, -1)
if state_of_interest == sCur[state_index]:
insert_row += [gisid]
inserted_oid = iCur.insertRow(insert_row)
# print 'Match:', state_of_interest, sCur[-1]
running_total += 1
gisid += 1
edit.stopOperation()
edit.stopEditing(True)
move_logger.info("Total|" + str(running_total))
del sCur, iCur
mc.write_gisid_seed(gisid)
def copy_fcs_by_state(logger_local_fcs, source_workspace, target_workspace, state_of_interest):
for fc in MC.get_gps_fcs(source_workspace):
print fc
sourceFC = os.path.join(source_workspace, fc)
targetFC = os.path.join(target_workspace, fc)
# MC.disable_editor_tracking(target_workspace, fc)
# MC.disable_spatial_index(target_workspace, fc)
arcpy.env.workspace = target_workspace
copy_fc(logger_local_fcs, target_workspace, sourceFC, targetFC, state_of_interest)
def assign_gisid_matches(preprod_workspace, prod_workspace, feature_class, preprod_final_list, prod_final_list, gisid,
p_fw):
pp_where = '"POINTRECORD_ID" IS NOT NULL AND "QASTATUS" = 600 AND "GISID" IS NULL'
pp_list = get_list_null_gisids(preprod_workspace, feature_class, pp_where)
p_where = '"POINTRECORD_ID" IS NOT NULL AND "GISID" IS NULL'
p_list = get_list_null_gisids(prod_workspace, feature_class, p_where)
# sort by the QA Status so the most important is always first
pp_list = sorted(pp_list, key=lambda x: (x[0], x[1], -x[2], -x[3]))
p_list = sorted(p_list, key=lambda x: (x[0], x[1], -x[2], -x[3]))
logger.info('|'.join(["Total_Queried", str(len(pp_list)), str(len(p_list))]))
# uniqueify
pp_uni = pp_list_2_dict_with_lists(pp_list)
p_uni = p_list # f7(p_list)
logger.info('|'.join(["Total_Uniqued", str(len(pp_uni)), str(len(p_uni))]))
local_count = 0
for p_item in p_uni:
if p_item[1] in pp_uni:
list_gisids_for_pp_point = pp_uni[p_item[1]]
if list_gisids_for_pp_point: # if empty, this is false
curr_pp_gisid = list_gisids_for_pp_point.pop(0) # pop off the top match
prod_final_list.append((p_item, gisid))
preprod_final_list.append((curr_pp_gisid, gisid))
print_tup2(p_fw, curr_pp_gisid, p_item, gisid)
gisid += 1
local_count += 1
logger.info('|'.join(["Total_Matched", str(local_count), str(local_count)]))
MC.write_gisid_seed(gisid)
return gisid, local_count
def copy_gps(log_name, source_workspace, target_workspace, state_of_interest):
logger = MC.setup_logging(log_name)
logger.info("Starting program...")
logger.info(state_of_interest)
logger.info(source_workspace + " --> " + target_workspace)
MC.arcfm_disable()
copy_fcs_by_state(logger, source_workspace, target_workspace, state_of_interest)
MC.arcfm_cleanup()
MC.send_finished_email(log_name)
logger.info("Finished program")
def copy_fc(logger_local_fc, target_workspace, source_fc, target_fc, fc_gps_state):
dsc = arcpy.Describe(source_fc)
fields = dsc.fields
fieldnames = [field.name for field in fields if
field.name != dsc.OIDFieldName and field.name != 'GLOBALID' and field.name != 'SHAPE']
fieldnames = fieldnames + ["SHAPE@"]
fieldnames_oid = fieldnames + [dsc.OIDFieldName]
where = """"GPS_STATE" ='{0}'""".format(fc_gps_state)
where += """ AND "OBJECTID" >= {0}"""
sql_clause = (None, 'ORDER BY "OBJECTID"')
running_total = 0
cursor_counter = 0
batch_size = 100000
run_again = True
start_oid = 0
# get the object id, query and sort by order id asc, keep the latest object id, then query ObjectID > that
while run_again:
edit = arcpy.da.Editor(target_workspace)
edit.startEditing(False, True)
edit.startOperation()
run_again = False
where_now = where.format(start_oid)
with arcpy.da.SearchCursor(source_fc, fieldnames_oid, where_now, None, False, sql_clause) as sCur:
with arcpy.da.InsertCursor(target_fc, fieldnames) as iCur:
for row in sCur:
if cursor_counter > batch_size:
cursor_counter = 0
run_again = True
start_oid = row[-1]
break
insert_row = MC.tuple_without(row, -1)
inserted_oid = iCur.insertRow(insert_row)
cursor_counter += 1
running_total += 1
if cursor_counter > 0 and cursor_counter % (batch_size / 10) == 0:
print running_total
edit.stopOperation()
edit.stopEditing(True)
logger_local_fc.info(source_fc + " | " + str(running_total))
def copy_line(copy_logger, target_workspace, source_fc, target_fc, fc_gps_state):
dsc = arcpy.Describe(source_fc)
fields = dsc.fields
fieldnames = [
field.name
for field in fields
if field.name != dsc.OIDFieldName and field.name != "GLOBALID" and field.name != "SHAPE"
]
fieldnames = fieldnames + ["SHAPE@"]
fieldnames_oid = fieldnames + ["OID@"]
# where = """"OBJECTID" >= {0}"""
sql_clause = (None, 'ORDER BY "OBJECTID"')
running_total = 0
edit = arcpy.da.Editor(target_workspace)
edit.startEditing(False, True)
edit.startOperation()
with arcpy.da.SearchCursor(source_fc, fieldnames_oid, "", None, False, sql_clause) as sCur:
with arcpy.da.InsertCursor(target_fc, fieldnames) as iCur:
for row in sCur:
# insert_row = MC.tuple_without(MC.tuple_without(row, -1), -1)
insert_row = mc.tuple_without(row, -1)
if row[-2] is None:
copy_logger.info(str(row[-1]) + " has None for Shape. Not copied.")
continue
if fc_gps_state == "OH" and row[-2].firstPoint.X < 4200000:
inserted_oid = iCur.insertRow(insert_row)
# print 'OH Match:', sCur[-1], sCur[-2].firstPoint.X, sCur[-2].firstPoint.Y
running_total += 1
elif fc_gps_state == "VA" and row[-2].firstPoint.X >= 4200000 and row[-2].firstPoint.Y < 6440000:
inserted_oid = iCur.insertRow(insert_row)
# print 'VA Match:', sCur[-1], sCur[-2].firstPoint.X, sCur[-2].firstPoint.Y
running_total += 1
elif fc_gps_state == "PA" and row[-2].firstPoint.X >= 4200000 and row[-2].firstPoint.Y >= 6440000:
inserted_oid = iCur.insertRow(insert_row)
# print 'PA Match:', sCur[-1], sCur[-2].firstPoint.X, sCur[-2].firstPoint.Y
running_total += 1
edit.stopOperation()
edit.stopEditing(True)
copy_logger.info("Total|" + str(running_total))
import Migrate2 as M2
import MigrationCommon as MC
#
# start of program
#
log_name = 'migrate2_PP'
logger = MC.setup_logging(log_name)
logger.info("Starting program...")
pp_workspace = MC.get_preprod_workspace()
results_file = "results/mig1_results.txt"
pp_dict = M2.load_dict(results_file, True)
logger.info(pp_workspace)
pp_where = '"POINTRECORD_ID" IS NOT NULL AND "QASTATUS" = 600 AND "GISID" IS NULL'
M2.update_fcs_gisids(logger, pp_workspace, MC.get_gps_fcs(pp_workspace), pp_dict, pp_where)
MC.send_finished_email(log_name)
logger.info("Finished program")
def copy_preprod(log_name, state_of_interest):
pp_workspace = MC.get_preprod_workspace()
r2d2_preprod = MC.get_r2d2_preprod_workspace(state_of_interest)
copy_gps(log_name, pp_workspace, r2d2_preprod, state_of_interest)
import arcpy
import MigrationCommon as MC
import CheckoutCommon as CC
def count_fc_state(feature_class):
if not arcpy.Exists(feature_class):
return -1
where = """"{0}" IS NOT NULL""".format(CC.find_state_field(feature_class))
return CC.count_fc_where(feature_class, where)
#
# start of program
#
log_name = 'checkout0'
logger = MC.setup_logging(log_name)
logger.info("Starting program...")
pp_workspace = MC.get_preprod_workspace()
p_workspace = MC.get_prod_workspace()
pp_dict = CC.count_fcs(pp_workspace, count_fc_state)
p_dict = CC.count_fcs(p_workspace, count_fc_state)
CC.write_r2d1_results(log_name, logger, pp_dict, p_dict)
logger.info("Finished program")
def disable_workspace(local_logger, workspace):
local_logger.info(workspace)
for fc in [x for x in cc.list_all_fcs(workspace) if x != 'NISOURCE.GPSFMEJobLog']:
if arcpy.Exists(fc):
local_logger.info(fc)
mc.disable_editor_tracking(workspace, fc)
for k in prod_shape_dict:
if k in pp_shape_dict:
for coord in prod_shape_dict[k]:
if coord not in pp_shape_dict[k]:
# print fc, k, coord, "has moved!"
for point in pp_point_dict[k]:
print ",".join(map(str, point)).replace("(", "").replace(")", "")
for point in prod_point_dict[k]:
print ",".join(map(str, point)).replace("(", "").replace(")", "")
pp_workspace = r"connections\z_PROD_xs08_sde.sde" # mc.get_preprod_workspace()
p_workspace = r"connections\z_PROD_xs09_sde.sde" # mc.get_prod_workspace()
# fc = "NISOURCE.GPS_CUSTOMERCONNECTION_G"
for fc in mc.get_gps_fcs(p_workspace):
find_diff(pp_workspace, p_workspace, fc)
# d = dict((k, v) for k, v in pp_d1.items() if len(v) > 1)
# for s in d:
# print s, d[s]
#
# print 'Prod now....'
# d = dict((k, v) for k, v in prod_d1.items() if len(v) > 1)
# for s in d:
# print s, d[s]
running_total += 1
gisid += 1
if cursor_counter > 0 and cursor_counter % 10000 == 0:
print cursor_counter
MC.write_gisid_seed(gisid)
edit.stopOperation()
edit.stopEditing(True)
logger.info(fc + "|" + str(running_total))
#
# start of program
#
log_name = 'migrate3'
logger = MC.setup_logging(log_name)
logger.info("Starting program...")
pp_workspace = MC.get_preprod_workspace()
p_workspace = MC.get_prod_workspace()
logger.info(pp_workspace)
assign_workspace_gisids(pp_workspace)
logger.info(p_workspace)
assign_workspace_gisids(p_workspace)
MC.send_finished_email(log_name)
logger.info("Finished program")
# start of program
#
print str(datetime.datetime.now())
# ArcFM licensing
import win32com.client
app = win32com.client.Dispatch("Miner.Framework.Dispatch.MMAppInitializeDispatch")
runtime = win32com.client.Dispatch("Miner.Framework.Dispatch.MMRuntimeEnvironmentDispatch")
au = win32com.client.Dispatch('Miner.Framework.Dispatch.MMAutoupdaterDispatch')
runtime.RuntimeMode = 0x4 # mmRuntimeModeArcServer
app.Initialize(0x5)
au.AutoUpdaterMode = 0x8
# end ArcFm licensing
r2d2_preprod = MC.get_r2d2_preprod_workspace_nisource('OH')
print r2d2_preprod
arcpy.env.workspace = r2d2_preprod
# arcpy.UnregisterAsVersioned_management("NISOURCE.GPSDataset", "NO_KEEP_EDIT", "COMPRESS_DEFAULT")
for fc in MC.get_gps_fcs(r2d2_preprod, True):
truncate_fc(r2d2_preprod, fc)
# arcpy.RegisterAsVersioned_management("NISOURCE.GPSDataset", "NO_EDITS_TO_BASE")
app.Shutdown
del app, runtime, au
print str(datetime.datetime.now())
# start of program
#
print str(datetime.datetime.now())
# ArcFM licensing
import win32com.client
app = win32com.client.Dispatch("Miner.Framework.Dispatch.MMAppInitializeDispatch")
runtime = win32com.client.Dispatch("Miner.Framework.Dispatch.MMRuntimeEnvironmentDispatch")
au = win32com.client.Dispatch('Miner.Framework.Dispatch.MMAutoupdaterDispatch')
runtime.RuntimeMode = 0x4 # mmRuntimeModeArcServer
app.Initialize(0x5)
au.AutoUpdaterMode = 0x8
# end ArcFm licensing
states = ["OH"]
for state in states:
print state
# r2d2_preprod = mc.get_r2d2_preprod_workspace(state)
# delete_fcs(r2d2_preprod)
r2d2_prod = mc.get_r2d2_prod_workspace(state)
delete_fcs(r2d2_prod)
app.Shutdown
del app, runtime, au
print str(datetime.datetime.now())
# start of program
#
print str(datetime.datetime.now())
# ArcFM licensing
import win32com.client
app = win32com.client.Dispatch("Miner.Framework.Dispatch.MMAppInitializeDispatch")
runtime = win32com.client.Dispatch("Miner.Framework.Dispatch.MMRuntimeEnvironmentDispatch")
au = win32com.client.Dispatch('Miner.Framework.Dispatch.MMAutoupdaterDispatch')
runtime.RuntimeMode = 0x4 # mmRuntimeModeArcServer
app.Initialize(0x5)
au.AutoUpdaterMode = 0x8
# end ArcFm licensing
r2d2_prod = MC.get_r2d2_prod_workspace('VA')
print r2d2_prod
arcpy.env.workspace = r2d2_prod
# fc = r"NISOURCE.GPS_GasValve"
# oidValueList = [r[0] for r in arcpy.da.SearchCursor(fc, ["OID@"])]
# oidValueList.sort()
# where = """"OBJECTID" >= {0}"""
for fc in MC.get_gps_fcs(r2d2_prod):
delete_fc(r2d2_prod, fc)
delete_fc(r2d2_prod, "NISOURCE.GPSWorkAreaPolygon")
preprod_final_list.append((curr_pp_gisid, gisid))
print_tup2(p_fw, curr_pp_gisid, p_item, gisid)
gisid += 1
local_count += 1
logger.info('|'.join(["Total_Matched", str(local_count), str(local_count)]))
MC.write_gisid_seed(gisid)
return gisid, local_count
#
# start of program
#
logger = MC.setup_logging('migrate1')
logger.info("Starting program...")
# setting this as it should start with this number for Migration 1
gisid_seed = 70000001 # MC.read_gisid_seed()
MC.write_gisid_seed(gisid_seed)
pp_workspace = MC.get_preprod_workspace()
p_workspace = MC.get_prod_workspace()
pp_final_list = []
p_final_list = []
total_matched_count = 0
with open("results/mig1_results.txt", "w+") as p_f:
def adding(workspace_local):
for add_value in add_these:
print ("Adding... " + add_value[1])
arcpy.AddCodedValueToDomain_management(workspace_local, "GPS_Polygon_Status", add_value[0], add_value[1])
def deleting(workspace_local):
for delete_value in delete_these:
print ("Removing... " + str(delete_value))
arcpy.DeleteCodedValueFromDomain_management(workspace_local, "GPS_Polygon_Status", delete_value)
# workspace = r'C:\Users\u906210\AppData\Roaming\ESRI\Desktop10.1\ArcCatalog\dg08_nisource.sde' # r'E:\Richard\FileGBs\tg13_test.gdb'
# arcpy.env.workspace = workspace
states = ["KY", "OH", "PA", "VA", "MA"]
for state in states:
pp_workspace = mc.get_r2d2_preprod_workspace_nisource(state)
print pp_workspace
arcpy.env.workspace = pp_workspace
adding(pp_workspace)
deleting(pp_workspace)
p_workspace = mc.get_r2d2_prod_workspace_nisource(state)
print p_workspace
arcpy.env.workspace = p_workspace
adding(p_workspace)
deleting(p_workspace)
import MigrationCommon as MC
logger = MC.setup_logging('checkout1')
logger.info("Starting program...")
with open("results/mig1_results.txt") as f:
num_lines = sum(1 for line in f)
logger.info('Results file length: ' + str(num_lines))
gisid_seed = MC.read_gisid_seed()
logger.info('GISID seed value: ' + str(gisid_seed))
logger.info('Difference between: ' + str(gisid_seed - num_lines))
logger.info("Finished program")
import MigrationCommon as MC
import CheckoutCommon as CC
import Checkout4 as C4
def count_r2d2_lines(state_of_interest):
pp_workspace = MC.get_r2d2_preprod_workspace(state_of_interest)
pp_lines = CC.list_lines_fcs(pp_workspace)
pp_dict = CC.count_list_fcs(pp_lines, C4.count_r2d2_fc_state)
return pp_dict
#
# start of program
#
if __name__ == "__main__":
log_name = 'checkout5'
logger = MC.setup_logging(log_name)
logger.info("Starting program...")
CC.write_rd2d2_by_state(log_name, logger, count_r2d2_lines)
logger.info("Finished program")
def count_r2d2_lines(state_of_interest):
pp_workspace = MC.get_r2d2_preprod_workspace(state_of_interest)
pp_lines = CC.list_lines_fcs(pp_workspace)
pp_dict = CC.count_list_fcs(pp_lines, C4.count_r2d2_fc_state)
return pp_dict
def assign_workspace_gisids(workspace):
for fc in MC.get_gps_fcs(workspace, True):
print fc
# MC.disable_editor_tracking(workspace, fc)
arcpy.env.workspace = workspace
assign_fc_gisids(workspace, fc)
edit.stopEditing(True)
move_logger.info("Total|" + str(running_total))
del sCur, iCur
mc.write_gisid_seed(gisid)
# time.sleep(10)
# MC.enable_editor_tracking(target_workspace, fc)
#
# start of program
#
log_name = 'migrate6'
logger = mc.setup_logging(log_name)
logger.info("Starting program...")
mc.arcfm_disable()
states = ["OH", "PA", "VA"]
for gps_state in states:
p_workspace = mc.get_prod_workspace()
r2d2_prod = mc.get_r2d2_prod_workspace(gps_state)
logger.info(p_workspace + "-->" + r2d2_prod)
move_workarea(logger, p_workspace, r2d2_prod, gps_state)
mc.arcfm_cleanup()
import CheckoutCommon as cc
def disable_workspace(local_logger, workspace):
local_logger.info(workspace)
for fc in [x for x in cc.list_all_fcs(workspace) if x != 'NISOURCE.GPSFMEJobLog']:
if arcpy.Exists(fc):
local_logger.info(fc)
mc.disable_editor_tracking(workspace, fc)
#
# start of program
#
if __name__ == "__main__":
logger = mc.setup_logging('disable_et')
states = ['OH', 'PA', 'VA']
logger.info("R2D1 GPS feature classes...")
disable_workspace(logger, mc.get_preprod_workspace_nisource())
disable_workspace(logger, mc.get_prod_workspace_nisource())
logger.info("R2D2 GPS feature classes...")
for state in states:
logger.info(state)
disable_workspace(logger, mc.get_r2d2_preprod_workspace_nisource(state))
logger.info("R2D2 GIS feature classes...")
for state in states:
logger.info(state)
disable_workspace(logger, mc.get_r2d2_prod_workspace_nisource(state))
