def assign_fc_gisids(workspace, fc): running_total = 0 cursor_counter = 0 batch_size = 100000 run_again = True while run_again: gisid = MC.read_gisid_seed() edit = arcpy.da.Editor(workspace) edit.startEditing(False, True) edit.startOperation() run_again = False # UpdateCursor (in_table, field_names, {where_clause}, {spatial_reference}, {explode_to_points}, {sql_clause}) with arcpy.da.UpdateCursor(fc, ["OID@", "GISID"], """"GISID" IS NULL OR "GISID" = 0""") as cursor: for row in cursor: if cursor_counter > batch_size: cursor_counter = 0 run_again = True break else: row[1] = gisid cursor.updateRow(row) cursor_counter += 1 running_total += 1 gisid += 1 if cursor_counter > 0 and cursor_counter % 10000 == 0: print cursor_counter MC.write_gisid_seed(gisid) edit.stopOperation() edit.stopEditing(True) logger.info(fc + "|" + str(running_total))
def clear_workspace_gisids(env): for fc in MC.get_gps_fcs(env, True): print fc MC.disable_editor_tracking(env, fc) arcpy.env.workspace = env clear_fc_gisids(env, fc) MC.enable_editor_tracking(env, fc)
def count_where_gisid_assigned(logname): logger = MC.setup_logging(logname) logger.info("Starting program...") pp_workspace = MC.get_preprod_workspace() p_workspace = MC.get_prod_workspace() pp_dict = CC.count_fcs(pp_workspace, CC.count_fc_gisid_assigned) p_dict = CC.count_fcs(p_workspace, CC.count_fc_gisid_assigned) CC.write_r2d1_results(logname, logger, pp_dict, p_dict) logger.info("Finished program")
def move_job_log(move_logger, source_workspace, target_workspace, state_of_interest): fc = "NISOURCE.GPSFMEJobLog" source_fc = os.path.join(source_workspace, fc) target_fc = os.path.join(target_workspace, fc) arcpy.env.workspace = target_workspace dsc = arcpy.Describe(source_fc) fields = dsc.fields fieldnames = [field.name for field in fields if field.name != dsc.OIDFieldName and field.name != 'GLOBALID' and field.name != 'SHAPE'] fieldnames_oid = fieldnames + ["OID@"] state_index = fieldnames.index("TCCORGNAME") sql_clause = (None, 'ORDER BY "OBJECTID"') running_total = 0 with arcpy.da.SearchCursor(source_fc, fieldnames_oid, "", None, False, sql_clause) as sCur: with arcpy.da.InsertCursor(target_fc, fieldnames) as iCur: for row in sCur: insert_row = mc.tuple_without(row, -1) if (state_of_interest == 'OH' and sCur[state_index].lower() == 'nisourcecoh') or ( state_of_interest == 'PA' and sCur[state_index].lower() == 'nisourcefre') or ( state_of_interest == 'VA' and sCur[state_index].lower() == 'nisourcecgv'): inserted_oid = iCur.insertRow(insert_row) # print 'Match:', state_of_interest, sCur[-1] running_total += 1 move_logger.info("Total|" + str(running_total))
def count_r2d2_wap(state_of_interest): p_workspace = MC.get_r2d2_prod_workspace(state_of_interest) arcpy.env.workspace = p_workspace p_wap = ['NISOURCE.GPSWorkAreaPolygon'] p_dict = CC.count_list_fcs(p_wap, C4.count_r2d2_fc_state) return p_dict
def delete_fcs(workspace_local): arcpy.env.workspace = workspace_local print workspace_local for fc in mc.get_gps_fcs(workspace_local, True): print fc if fc != 'NISOURCE.GPS_CustomerConnection_G': delete_fc(workspace_local, fc)
def move_workarea(move_logger, source_workspace, target_workspace, state_of_interest): fc = "NISOURCE.GPSWorkAreaPolygon" source_fc = os.path.join(source_workspace, fc) target_fc = os.path.join(target_workspace, fc) # MC.disable_editor_tracking(target_workspace, fc) arcpy.env.workspace = target_workspace dsc = arcpy.Describe(source_fc) fields = dsc.fields fieldnames = [field.name for field in fields if field.name != dsc.OIDFieldName and field.name != 'GLOBALID' and field.name != 'SHAPE'] fieldnames = fieldnames + ["SHAPE@"] fieldnames_oid = fieldnames + ["OID@"] fieldnames_insert = fieldnames + ["GISID"] state_index = fieldnames.index("WORKAREASTATE") sql_clause = (None, 'ORDER BY "OBJECTID"') running_total = 0 gisid = mc.read_gisid_seed() edit = arcpy.da.Editor(target_workspace) edit.startEditing(False, True) edit.startOperation() with arcpy.da.SearchCursor(source_fc, fieldnames_oid, "", None, False, sql_clause) as sCur: with arcpy.da.InsertCursor(target_fc, fieldnames_insert) as iCur: for row in sCur: insert_row = mc.tuple_without(row, -1) if state_of_interest == sCur[state_index]: insert_row += [gisid] inserted_oid = iCur.insertRow(insert_row) # print 'Match:', state_of_interest, sCur[-1] running_total += 1 gisid += 1 edit.stopOperation() edit.stopEditing(True) move_logger.info("Total|" + str(running_total)) del sCur, iCur mc.write_gisid_seed(gisid)
def copy_fcs_by_state(logger_local_fcs, source_workspace, target_workspace, state_of_interest): for fc in MC.get_gps_fcs(source_workspace): print fc sourceFC = os.path.join(source_workspace, fc) targetFC = os.path.join(target_workspace, fc) # MC.disable_editor_tracking(target_workspace, fc) # MC.disable_spatial_index(target_workspace, fc) arcpy.env.workspace = target_workspace copy_fc(logger_local_fcs, target_workspace, sourceFC, targetFC, state_of_interest)
def assign_gisid_matches(preprod_workspace, prod_workspace, feature_class, preprod_final_list, prod_final_list, gisid, p_fw): pp_where = '"POINTRECORD_ID" IS NOT NULL AND "QASTATUS" = 600 AND "GISID" IS NULL' pp_list = get_list_null_gisids(preprod_workspace, feature_class, pp_where) p_where = '"POINTRECORD_ID" IS NOT NULL AND "GISID" IS NULL' p_list = get_list_null_gisids(prod_workspace, feature_class, p_where) # sort by the QA Status so the most important is always first pp_list = sorted(pp_list, key=lambda x: (x[0], x[1], -x[2], -x[3])) p_list = sorted(p_list, key=lambda x: (x[0], x[1], -x[2], -x[3])) logger.info('|'.join(["Total_Queried", str(len(pp_list)), str(len(p_list))])) # uniqueify pp_uni = pp_list_2_dict_with_lists(pp_list) p_uni = p_list # f7(p_list) logger.info('|'.join(["Total_Uniqued", str(len(pp_uni)), str(len(p_uni))])) local_count = 0 for p_item in p_uni: if p_item[1] in pp_uni: list_gisids_for_pp_point = pp_uni[p_item[1]] if list_gisids_for_pp_point: # if empty, this is false curr_pp_gisid = list_gisids_for_pp_point.pop(0) # pop off the top match prod_final_list.append((p_item, gisid)) preprod_final_list.append((curr_pp_gisid, gisid)) print_tup2(p_fw, curr_pp_gisid, p_item, gisid) gisid += 1 local_count += 1 logger.info('|'.join(["Total_Matched", str(local_count), str(local_count)])) MC.write_gisid_seed(gisid) return gisid, local_count
def copy_gps(log_name, source_workspace, target_workspace, state_of_interest): logger = MC.setup_logging(log_name) logger.info("Starting program...") logger.info(state_of_interest) logger.info(source_workspace + " --> " + target_workspace) MC.arcfm_disable() copy_fcs_by_state(logger, source_workspace, target_workspace, state_of_interest) MC.arcfm_cleanup() MC.send_finished_email(log_name) logger.info("Finished program")
def copy_fc(logger_local_fc, target_workspace, source_fc, target_fc, fc_gps_state): dsc = arcpy.Describe(source_fc) fields = dsc.fields fieldnames = [field.name for field in fields if field.name != dsc.OIDFieldName and field.name != 'GLOBALID' and field.name != 'SHAPE'] fieldnames = fieldnames + ["SHAPE@"] fieldnames_oid = fieldnames + [dsc.OIDFieldName] where = """"GPS_STATE" ='{0}'""".format(fc_gps_state) where += """ AND "OBJECTID" >= {0}""" sql_clause = (None, 'ORDER BY "OBJECTID"') running_total = 0 cursor_counter = 0 batch_size = 100000 run_again = True start_oid = 0 # get the object id, query and sort by order id asc, keep the latest object id, then query ObjectID > that while run_again: edit = arcpy.da.Editor(target_workspace) edit.startEditing(False, True) edit.startOperation() run_again = False where_now = where.format(start_oid) with arcpy.da.SearchCursor(source_fc, fieldnames_oid, where_now, None, False, sql_clause) as sCur: with arcpy.da.InsertCursor(target_fc, fieldnames) as iCur: for row in sCur: if cursor_counter > batch_size: cursor_counter = 0 run_again = True start_oid = row[-1] break insert_row = MC.tuple_without(row, -1) inserted_oid = iCur.insertRow(insert_row) cursor_counter += 1 running_total += 1 if cursor_counter > 0 and cursor_counter % (batch_size / 10) == 0: print running_total edit.stopOperation() edit.stopEditing(True) logger_local_fc.info(source_fc + " | " + str(running_total))
def copy_line(copy_logger, target_workspace, source_fc, target_fc, fc_gps_state): dsc = arcpy.Describe(source_fc) fields = dsc.fields fieldnames = [ field.name for field in fields if field.name != dsc.OIDFieldName and field.name != "GLOBALID" and field.name != "SHAPE" ] fieldnames = fieldnames + ["SHAPE@"] fieldnames_oid = fieldnames + ["OID@"] # where = """"OBJECTID" >= {0}""" sql_clause = (None, 'ORDER BY "OBJECTID"') running_total = 0 edit = arcpy.da.Editor(target_workspace) edit.startEditing(False, True) edit.startOperation() with arcpy.da.SearchCursor(source_fc, fieldnames_oid, "", None, False, sql_clause) as sCur: with arcpy.da.InsertCursor(target_fc, fieldnames) as iCur: for row in sCur: # insert_row = MC.tuple_without(MC.tuple_without(row, -1), -1) insert_row = mc.tuple_without(row, -1) if row[-2] is None: copy_logger.info(str(row[-1]) + " has None for Shape. Not copied.") continue if fc_gps_state == "OH" and row[-2].firstPoint.X < 4200000: inserted_oid = iCur.insertRow(insert_row) # print 'OH Match:', sCur[-1], sCur[-2].firstPoint.X, sCur[-2].firstPoint.Y running_total += 1 elif fc_gps_state == "VA" and row[-2].firstPoint.X >= 4200000 and row[-2].firstPoint.Y < 6440000: inserted_oid = iCur.insertRow(insert_row) # print 'VA Match:', sCur[-1], sCur[-2].firstPoint.X, sCur[-2].firstPoint.Y running_total += 1 elif fc_gps_state == "PA" and row[-2].firstPoint.X >= 4200000 and row[-2].firstPoint.Y >= 6440000: inserted_oid = iCur.insertRow(insert_row) # print 'PA Match:', sCur[-1], sCur[-2].firstPoint.X, sCur[-2].firstPoint.Y running_total += 1 edit.stopOperation() edit.stopEditing(True) copy_logger.info("Total|" + str(running_total))
import Migrate2 as M2 import MigrationCommon as MC # # start of program # log_name = 'migrate2_PP' logger = MC.setup_logging(log_name) logger.info("Starting program...") pp_workspace = MC.get_preprod_workspace() results_file = "results/mig1_results.txt" pp_dict = M2.load_dict(results_file, True) logger.info(pp_workspace) pp_where = '"POINTRECORD_ID" IS NOT NULL AND "QASTATUS" = 600 AND "GISID" IS NULL' M2.update_fcs_gisids(logger, pp_workspace, MC.get_gps_fcs(pp_workspace), pp_dict, pp_where) MC.send_finished_email(log_name) logger.info("Finished program")
def copy_preprod(log_name, state_of_interest): pp_workspace = MC.get_preprod_workspace() r2d2_preprod = MC.get_r2d2_preprod_workspace(state_of_interest) copy_gps(log_name, pp_workspace, r2d2_preprod, state_of_interest)
import arcpy import MigrationCommon as MC import CheckoutCommon as CC def count_fc_state(feature_class): if not arcpy.Exists(feature_class): return -1 where = """"{0}" IS NOT NULL""".format(CC.find_state_field(feature_class)) return CC.count_fc_where(feature_class, where) # # start of program # log_name = 'checkout0' logger = MC.setup_logging(log_name) logger.info("Starting program...") pp_workspace = MC.get_preprod_workspace() p_workspace = MC.get_prod_workspace() pp_dict = CC.count_fcs(pp_workspace, count_fc_state) p_dict = CC.count_fcs(p_workspace, count_fc_state) CC.write_r2d1_results(log_name, logger, pp_dict, p_dict) logger.info("Finished program")
def disable_workspace(local_logger, workspace): local_logger.info(workspace) for fc in [x for x in cc.list_all_fcs(workspace) if x != 'NISOURCE.GPSFMEJobLog']: if arcpy.Exists(fc): local_logger.info(fc) mc.disable_editor_tracking(workspace, fc)
for k in prod_shape_dict: if k in pp_shape_dict: for coord in prod_shape_dict[k]: if coord not in pp_shape_dict[k]: # print fc, k, coord, "has moved!" for point in pp_point_dict[k]: print ",".join(map(str, point)).replace("(", "").replace(")", "") for point in prod_point_dict[k]: print ",".join(map(str, point)).replace("(", "").replace(")", "") pp_workspace = r"connections\z_PROD_xs08_sde.sde" # mc.get_preprod_workspace() p_workspace = r"connections\z_PROD_xs09_sde.sde" # mc.get_prod_workspace() # fc = "NISOURCE.GPS_CUSTOMERCONNECTION_G" for fc in mc.get_gps_fcs(p_workspace): find_diff(pp_workspace, p_workspace, fc) # d = dict((k, v) for k, v in pp_d1.items() if len(v) > 1) # for s in d: # print s, d[s] # # print 'Prod now....' # d = dict((k, v) for k, v in prod_d1.items() if len(v) > 1) # for s in d: # print s, d[s]
running_total += 1 gisid += 1 if cursor_counter > 0 and cursor_counter % 10000 == 0: print cursor_counter MC.write_gisid_seed(gisid) edit.stopOperation() edit.stopEditing(True) logger.info(fc + "|" + str(running_total)) # # start of program # log_name = 'migrate3' logger = MC.setup_logging(log_name) logger.info("Starting program...") pp_workspace = MC.get_preprod_workspace() p_workspace = MC.get_prod_workspace() logger.info(pp_workspace) assign_workspace_gisids(pp_workspace) logger.info(p_workspace) assign_workspace_gisids(p_workspace) MC.send_finished_email(log_name) logger.info("Finished program")
# start of program # print str(datetime.datetime.now()) # ArcFM licensing import win32com.client app = win32com.client.Dispatch("Miner.Framework.Dispatch.MMAppInitializeDispatch") runtime = win32com.client.Dispatch("Miner.Framework.Dispatch.MMRuntimeEnvironmentDispatch") au = win32com.client.Dispatch('Miner.Framework.Dispatch.MMAutoupdaterDispatch') runtime.RuntimeMode = 0x4 # mmRuntimeModeArcServer app.Initialize(0x5) au.AutoUpdaterMode = 0x8 # end ArcFm licensing r2d2_preprod = MC.get_r2d2_preprod_workspace_nisource('OH') print r2d2_preprod arcpy.env.workspace = r2d2_preprod # arcpy.UnregisterAsVersioned_management("NISOURCE.GPSDataset", "NO_KEEP_EDIT", "COMPRESS_DEFAULT") for fc in MC.get_gps_fcs(r2d2_preprod, True): truncate_fc(r2d2_preprod, fc) # arcpy.RegisterAsVersioned_management("NISOURCE.GPSDataset", "NO_EDITS_TO_BASE") app.Shutdown del app, runtime, au print str(datetime.datetime.now())
# start of program # print str(datetime.datetime.now()) # ArcFM licensing import win32com.client app = win32com.client.Dispatch("Miner.Framework.Dispatch.MMAppInitializeDispatch") runtime = win32com.client.Dispatch("Miner.Framework.Dispatch.MMRuntimeEnvironmentDispatch") au = win32com.client.Dispatch('Miner.Framework.Dispatch.MMAutoupdaterDispatch') runtime.RuntimeMode = 0x4 # mmRuntimeModeArcServer app.Initialize(0x5) au.AutoUpdaterMode = 0x8 # end ArcFm licensing states = ["OH"] for state in states: print state # r2d2_preprod = mc.get_r2d2_preprod_workspace(state) # delete_fcs(r2d2_preprod) r2d2_prod = mc.get_r2d2_prod_workspace(state) delete_fcs(r2d2_prod) app.Shutdown del app, runtime, au print str(datetime.datetime.now())
# start of program # print str(datetime.datetime.now()) # ArcFM licensing import win32com.client app = win32com.client.Dispatch("Miner.Framework.Dispatch.MMAppInitializeDispatch") runtime = win32com.client.Dispatch("Miner.Framework.Dispatch.MMRuntimeEnvironmentDispatch") au = win32com.client.Dispatch('Miner.Framework.Dispatch.MMAutoupdaterDispatch') runtime.RuntimeMode = 0x4 # mmRuntimeModeArcServer app.Initialize(0x5) au.AutoUpdaterMode = 0x8 # end ArcFm licensing r2d2_prod = MC.get_r2d2_prod_workspace('VA') print r2d2_prod arcpy.env.workspace = r2d2_prod # fc = r"NISOURCE.GPS_GasValve" # oidValueList = [r[0] for r in arcpy.da.SearchCursor(fc, ["OID@"])] # oidValueList.sort() # where = """"OBJECTID" >= {0}""" for fc in MC.get_gps_fcs(r2d2_prod): delete_fc(r2d2_prod, fc) delete_fc(r2d2_prod, "NISOURCE.GPSWorkAreaPolygon")
preprod_final_list.append((curr_pp_gisid, gisid)) print_tup2(p_fw, curr_pp_gisid, p_item, gisid) gisid += 1 local_count += 1 logger.info('|'.join(["Total_Matched", str(local_count), str(local_count)])) MC.write_gisid_seed(gisid) return gisid, local_count # # start of program # logger = MC.setup_logging('migrate1') logger.info("Starting program...") # setting this as it should start with this number for Migration 1 gisid_seed = 70000001 # MC.read_gisid_seed() MC.write_gisid_seed(gisid_seed) pp_workspace = MC.get_preprod_workspace() p_workspace = MC.get_prod_workspace() pp_final_list = [] p_final_list = [] total_matched_count = 0 with open("results/mig1_results.txt", "w+") as p_f:
def adding(workspace_local): for add_value in add_these: print ("Adding... " + add_value[1]) arcpy.AddCodedValueToDomain_management(workspace_local, "GPS_Polygon_Status", add_value[0], add_value[1]) def deleting(workspace_local): for delete_value in delete_these: print ("Removing... " + str(delete_value)) arcpy.DeleteCodedValueFromDomain_management(workspace_local, "GPS_Polygon_Status", delete_value) # workspace = r'C:\Users\u906210\AppData\Roaming\ESRI\Desktop10.1\ArcCatalog\dg08_nisource.sde' # r'E:\Richard\FileGBs\tg13_test.gdb' # arcpy.env.workspace = workspace states = ["KY", "OH", "PA", "VA", "MA"] for state in states: pp_workspace = mc.get_r2d2_preprod_workspace_nisource(state) print pp_workspace arcpy.env.workspace = pp_workspace adding(pp_workspace) deleting(pp_workspace) p_workspace = mc.get_r2d2_prod_workspace_nisource(state) print p_workspace arcpy.env.workspace = p_workspace adding(p_workspace) deleting(p_workspace)
import MigrationCommon as MC logger = MC.setup_logging('checkout1') logger.info("Starting program...") with open("results/mig1_results.txt") as f: num_lines = sum(1 for line in f) logger.info('Results file length: ' + str(num_lines)) gisid_seed = MC.read_gisid_seed() logger.info('GISID seed value: ' + str(gisid_seed)) logger.info('Difference between: ' + str(gisid_seed - num_lines)) logger.info("Finished program")
import MigrationCommon as MC import CheckoutCommon as CC import Checkout4 as C4 def count_r2d2_lines(state_of_interest): pp_workspace = MC.get_r2d2_preprod_workspace(state_of_interest) pp_lines = CC.list_lines_fcs(pp_workspace) pp_dict = CC.count_list_fcs(pp_lines, C4.count_r2d2_fc_state) return pp_dict # # start of program # if __name__ == "__main__": log_name = 'checkout5' logger = MC.setup_logging(log_name) logger.info("Starting program...") CC.write_rd2d2_by_state(log_name, logger, count_r2d2_lines) logger.info("Finished program")
def count_r2d2_lines(state_of_interest): pp_workspace = MC.get_r2d2_preprod_workspace(state_of_interest) pp_lines = CC.list_lines_fcs(pp_workspace) pp_dict = CC.count_list_fcs(pp_lines, C4.count_r2d2_fc_state) return pp_dict
def assign_workspace_gisids(workspace): for fc in MC.get_gps_fcs(workspace, True): print fc # MC.disable_editor_tracking(workspace, fc) arcpy.env.workspace = workspace assign_fc_gisids(workspace, fc)
edit.stopEditing(True) move_logger.info("Total|" + str(running_total)) del sCur, iCur mc.write_gisid_seed(gisid) # time.sleep(10) # MC.enable_editor_tracking(target_workspace, fc) # # start of program # log_name = 'migrate6' logger = mc.setup_logging(log_name) logger.info("Starting program...") mc.arcfm_disable() states = ["OH", "PA", "VA"] for gps_state in states: p_workspace = mc.get_prod_workspace() r2d2_prod = mc.get_r2d2_prod_workspace(gps_state) logger.info(p_workspace + "-->" + r2d2_prod) move_workarea(logger, p_workspace, r2d2_prod, gps_state) mc.arcfm_cleanup()
import CheckoutCommon as cc def disable_workspace(local_logger, workspace): local_logger.info(workspace) for fc in [x for x in cc.list_all_fcs(workspace) if x != 'NISOURCE.GPSFMEJobLog']: if arcpy.Exists(fc): local_logger.info(fc) mc.disable_editor_tracking(workspace, fc) # # start of program # if __name__ == "__main__": logger = mc.setup_logging('disable_et') states = ['OH', 'PA', 'VA'] logger.info("R2D1 GPS feature classes...") disable_workspace(logger, mc.get_preprod_workspace_nisource()) disable_workspace(logger, mc.get_prod_workspace_nisource()) logger.info("R2D2 GPS feature classes...") for state in states: logger.info(state) disable_workspace(logger, mc.get_r2d2_preprod_workspace_nisource(state)) logger.info("R2D2 GIS feature classes...") for state in states: logger.info(state) disable_workspace(logger, mc.get_r2d2_prod_workspace_nisource(state))