def addStandardMosaicDatasetFields(md_path): arcpy.AddMessage("Adding fields to Mosaic Dataset '{}'".format(md_path)) # Add the required metadata fields to the Master/Project Mosaic Dataset # Utility.addAndCalcFieldText(dataset_path=md_path, field_name=CMDRConfig.PROJECT_ID, field_length="100", field_alias=CMDRConfig.PROJECT_ID.replace("_", " "), add_index=True) Utility.addAndCalcFieldDate(dataset_path=md_path, field_name=CMDRConfig.PROJECT_DATE, field_alias=CMDRConfig.PROJECT_DATE.replace( "_", " "), add_index=True) Utility.addAndCalcFieldText(dataset_path=md_path, field_name=CMDRConfig.RASTER_PATH, field_length="800", field_alias=CMDRConfig.RASTER_PATH.replace( "_", " "), add_index=True) Utility.addAndCalcFieldText(dataset_path=md_path, field_name=CMDRConfig.PROJECT_SOURCE, field_length="20", field_alias=CMDRConfig.PROJECT_SOURCE.replace( "_", " "), add_index=True) # Utility.addAndCalcFieldText(dataset_path=md_path, field_name=CMDRConfig.PROJECT_SR_XY_NAME, field_length="100", field_alias=CMDRConfig.PROJECT_SR_XY_NAME.replace("_", " "), add_index=True) # Utility.addAndCalcFieldText(dataset_path=md_path, field_name=CMDRConfig.PROJECT_SR_XY_UNITS, field_length="20", field_alias=CMDRConfig.PROJECT_SR_XY_UNITS.replace("_", " "), add_index=True) # Utility.addAndCalcFieldText(dataset_path=md_path, field_name=CMDRConfig.PROJECT_SR_XY_CODE, field_length="100", field_alias=CMDRConfig.PROJECT_SR_XY_CODE.replace("_", " "), add_index=True) # Utility.addAndCalcFieldText(dataset_path=md_path, field_name=CMDRConfig.PROJECT_SR_Z_NAME, field_length="100", field_alias=CMDRConfig.PROJECT_SR_Z_NAME.replace("_", " "), add_index=True) # Utility.addAndCalcFieldText(dataset_path=md_path, field_name=CMDRConfig.PROJECT_SR_Z_UNITS, field_length="100", field_alias=CMDRConfig.PROJECT_SR_Z_UNITS.replace("_", " "), add_index=True) # Utility.addAndCalcFieldText(dataset_path=md_path, field_name=CMDRConfig.FILE_NAME, field_length="100", field_alias=CMDRConfig.FILE_NAME.replace("_", " "), add_index=True) arcpy.EnableEditorTracking_management( in_dataset=md_path, creator_field="created_user", creation_date_field="created_date", last_editor_field="last_edited_user", last_edit_date_field="last_edited_date", add_fields="ADD_FIELDS", record_dates_in="UTC") Utility.addToolMessages()
def generate_contour(md, cont_int, contUnits, rasterUnits, smooth_tol, scratch_path, proc_dict): from datetime import datetime name = proc_dict[0] index = str(proc_dict[1][2]) arcpy.AddMessage("Checking out licenses") arcpy.CheckOutExtension("3D") arcpy.CheckOutExtension("Spatial") created = False tries = 0 outOfMemory = False vertexLimit = 100 featureCount = 25000 while not created and tries <= TRIES_ALLOWED: tries = tries + 1 try: a = datetime.now() aa = a Utility.setArcpyEnv(True) arcpy.AddMessage('STARTING ' + name + ' ' + index + ': Generating Contours') buff_poly = proc_dict[1][0] clip_poly = proc_dict[1][1] #arcpy.AddMessage("\t{}: Buffer Poly '{}'".format(name, buff_poly)) #arcpy.AddMessage("\t{}: Clip Poly '{}'".format(name, clip_poly)) arcpy.env.extent = buff_poly.extent arcpy.env.XYResolution = "0.0001 Meters" if USE_FEATURE_CLASS: gdbName = "{}.gdb".format(name) fileExtension = "" workspace = os.path.join(scratch_path, gdbName) else: fileExtension = ".shp" workspace = os.path.join(scratch_path, name) if not arcpy.Exists(workspace): # Don't delete if it exists, keep our previous work to save time if USE_FEATURE_CLASS: arcpy.AddMessage( "\nCreating Workspace GDB: {0}".format(workspace)) arcpy.CreateFileGDB_management(scratch_path, gdbName, out_version="CURRENT") else: os.mkdir(workspace) arcpy.env.workspace = workspace a = doTime( a, '\t' + name + ' ' + index + ': Created scratch workspace' + workspace) focal2_path = md md_desc = arcpy.Describe(md) if not md_desc.referenced: arcpy.AddError( "\t{}: ERROR Referenced Mosaic not found '{}'".format( name, focal2_path)) arcpy.AddMessage("\t{}: Referenced Mosaic found '{}'".format( name, focal2_path)) base_name = 'O08_BaseCont_{}{}'.format(name, fileExtension) base_contours = os.path.join(workspace, base_name) outRasterLayer = "outRasterLayer" if not arcpy.Exists(base_contours): arcpy.MakeRasterLayer_management( in_raster=focal2_path, out_rasterlayer=outRasterLayer) Functions.Contour(outRasterLayer, base_contours, int(cont_int)) a = doTime( a, '\t' + name + ' ' + index + ': Contoured to ' + base_contours) simple_contours = os.path.join( workspace, 'O09_SimpleCont_{}{}'.format(name, fileExtension)) if outOfMemory: diced_contours = os.path.join( workspace, 'O08ADicedCont_{}{}'.format(name, fileExtension)) arcpy.Dice_management(base_contours, diced_contours, vertexLimit) a = doTime( a, '\t' + name + ' ' + index + ': Diced to ' + diced_contours) cartographic_partitions = os.path.join( workspace, 'Carto_Partitions_{}{}'.format(name, fileExtension)) arcpy.CreateCartographicPartitions_cartography( diced_contours, cartographic_partitions, featureCount) a = doTime( a, '\t' + name + ' ' + index + ': Created Cartographic Partitions at ' + cartographic_partitions) arcpy.env.cartographicPartitions = cartographic_partitions base_contours = diced_contours if arcpy.Exists(simple_contours): arcpy.Delete_management(simple_contours) if not arcpy.Exists(simple_contours): ca.SimplifyLine(base_contours, simple_contours, "POINT_REMOVE", "0.000001 DecimalDegrees", "FLAG_ERRORS", "NO_KEEP", "NO_CHECK") a = doTime( a, '\t' + name + ' ' + index + ': Simplified to ' + simple_contours) if rasterUnits == "Foot" or rasterUnits == "FT": maxShapeLength = 16.404 elif rasterUnits == "Meter" or rasterUnits == "MT": maxShapeLength = 5 else: maxShapeLength = 0 # BJN Need to add Shape_Length attribute to shapefile & calculate length if USE_FEATURE_CLASS = False if not USE_FEATURE_CLASS: SHAPE_LENGTH = 'Length' arcpy.AddField_management(simple_contours, SHAPE_LENGTH, 'Double') arcpy.CalculateField_management(simple_contours, SHAPE_LENGTH, '!shape.length!', 'PYTHON_9.3') else: SHAPE_LENGTH = 'Shape_Length' greaterThan2MetersSelection = 'greaterThan2MetersSelection' #BJN arcpy.MakeFeatureLayer_management( simple_contours, greaterThan2MetersSelection, "{} > {}".format(SHAPE_LENGTH, maxShapeLength)) # TODO: Select anything under 2 meters in length to a new 'small_contours' feature class # Delete the selection from the simple_contours # Delete any small contours snippets that are within 2 meters of the tile boundary # Run Feature to Point on the small contours and use the output in our contour map and service smooth_contours = os.path.join( workspace, 'O10_SmoothCont_{}{}'.format(name, fileExtension)) if not arcpy.Exists(smooth_contours): ca.SmoothLine(greaterThan2MetersSelection, smooth_contours, "PAEK", "{} DecimalDegrees".format(smooth_tol), "", "NO_CHECK") a = doTime( a, '\t' + name + ' ' + index + ': Smoothed to ' + smooth_contours) # put this up one level to avoid re-processing all of above if something goes wrong below clip_workspace = os.path.split(workspace)[0] clip_contours = os.path.join( clip_workspace, 'O11_ClipCont_{}.shp'.format(name)) #BJN if not arcpy.Exists(clip_contours): arcpy.Clip_analysis(in_features=smooth_contours, clip_features=clip_poly, out_feature_class=clip_contours) a = doTime( a, '\t' + name + ' ' + index + ': Clipped to ' + clip_contours) arcpy.RepairGeometry_management(in_features=clip_contours, delete_null="DELETE_NULL") Utility.addAndCalcFieldLong( dataset_path=clip_contours, field_name="CTYPE", field_value="getType( !CONTOUR! )", code_block= "def getType(contour):\n\n type = 2\n\n if contour%10 == 0:\n\n type = 10\n\n if contour%20 == 0:\n\n type = 20\n\n if contour%50 == 0:\n type = 50\n if contour%100 == 0:\n type = 100\n if contour%500 == 0:\n type = 500\n if contour%1000 == 0:\n type = 1000\n if contour%5000 == 0:\n type = 5000\n return type", add_index=False) Utility.addAndCalcFieldLong( dataset_path=clip_contours, field_name="INDEX", field_value="getType( !CONTOUR! )", code_block= "def getType(contour):\n\n type = 0\n\n if contour%" + str(int(cont_int * 5)) + " == 0:\n\n type = 1\n return type", add_index=False) # Utility.addAndCalcFieldText(dataset_path=clip_contours, # field_name="LastMergedFC", # field_length=100, # field_value=name, # add_index=False) # Utility.addAndCalcFieldText(dataset_path=clip_contours, # field_name="ValidationCheck", # field_length=100, # field_value='"'+name+'"', # add_index=False) Utility.addAndCalcFieldText(dataset_path=clip_contours, field_name="UNITS", field_length=20, field_value='"' + CONTOUR_UNIT + '"', add_index=False) Utility.addAndCalcFieldText(dataset_path=clip_contours, field_name="name", field_length=79, field_value='"' + name + '"', add_index=False) a = doTime( a, '\t' + name + ' ' + index + ': Added fields to ' + clip_contours) try: arcpy.DeleteField_management( in_table=clip_contours, drop_field="ID;InLine_FID;SimLnFlag;MaxSimpTol;MinSimpTol") a = doTime( a, '\t' + name + ' ' + index + ': Deleted fields from ' + clip_contours) except: pass doTime(aa, 'FINISHED ' + name + ' ' + index) created = True except arcpy.ExecuteError as exeErr: errorCode = str(exeErr).split(':')[0] if errorCode == 'ERROR 000426': if tries > 1 and outOfMemory: vertexLimit *= 0.75 featureCount *= 0.75 outOfMemory = True arcpy.AddMessage('\t\tProcess Dropped: ' + name + ' ' + index) arcpy.AddMessage('\t\tException: ' + str(exeErr)) if tries > TRIES_ALLOWED: arcpy.AddError('Too many tries, Dropped: {}'.format(name)) except Exception as e: arcpy.AddMessage('\t\tProcess Dropped: ' + name + ' ' + index) arcpy.AddMessage('\t\tException: ' + str(e)) if tries > TRIES_ALLOWED: arcpy.AddError('Too many tries, Dropped: {} {}'.format( name, index)) try: arcpy.AddMessage("Checking in licenses") arcpy.CheckInExtension("3D") arcpy.CheckInExtension("Spatial") except: pass
def generate_contour(md, cont_int, contUnits, rasterUnits, smooth_tol, scratch_path, proc_dict): name = proc_dict[0] index = str(proc_dict[1][2]) arcpy.AddMessage("Checking out licenses") arcpy.CheckOutExtension("3D") arcpy.CheckOutExtension("Spatial") created = False tries = 0 while not created and tries <= TRIES_ALLOWED: tries = tries + 1 try: a = datetime.now() aa = a Utility.setArcpyEnv(True) arcpy.AddMessage('STARTING ' + name + ' ' + index + ': Generating Contours') buff_poly = proc_dict[1][0] clip_poly = proc_dict[1][1] #arcpy.AddMessage("\t{}: Buffer Poly '{}'".format(name, buff_poly)) #arcpy.AddMessage("\t{}: Clip Poly '{}'".format(name, clip_poly)) arcpy.env.extent = buff_poly.extent workspace = os.path.join(scratch_path, name) if not os.path.exists(workspace): # Don't delete if it exists, keep our previous work to save time os.mkdir(workspace) arcpy.env.workspace = workspace a = doTime(a, '\t' + name + ' ' + index + ': Created scratch workspace' + workspace) focal2_path = md md_desc = arcpy.Describe(md) if not md_desc.referenced: arcpy.AddError("\t{}: ERROR Referenced Mosaic not found '{}'".format(name, focal2_path)) ## md_layer = arcpy.MakeMosaicLayer_management(in_mosaic_dataset=md, out_mosaic_layer="DTM_MosaicLayer", where_clause="TypeID = 1", template=buff_poly.extent) ## a = doTime(a, "\t" + name + ": Created mosaic layer for primary images") ## ## divide1_name = 'O01_Divide1_' + name + '.tif' ## divide1_path = os.path.join(workspace, divide1_name) ## if not os.path.exists(divide1_path): ## arcpy.MakeRasterLayer_management(in_raster=md_layer, out_rasterlayer=divide1_name) ## # @TODO: Clean up the unit conversion here. output will ALWAYS be US Survey Feet ### contUnits = contUnits.upper() ## contUnits = "FOOT_US" ## rasterUnits = rasterUnits.upper() ## outDivide1 = Functions.Divide(divide1_name, 1.0) ### if contUnits.find("METERS") >= 0 or contUnits.find("MT") >= 0: ### contUnits = "METER" ### elif contUnits.find("FOOT") >= 0 or contUnits.find("FEET") >= 0 or contUnits.find("FT") >= 0: ### contUnits = "FOOT_INTL" ### if contUnits.find("US") >= 0 or contUnits.find("SURVEY") >= 0: ### contUnits = "FOOT_US" ### #### if rasterUnits.find("METERS") >= 0 or rasterUnits.find("MT") >= 0: #### rasterUnits = "METER" #### elif rasterUnits.find("FOOT") >= 0 or rasterUnits.find("FEET") >= 0 or rasterUnits.find("FT") >= 0: #### rasterUnits = "FOOT_INTL" #### if rasterUnits.find("US") >= 0 or rasterUnits.find("SURVEY") >= 0: #### rasterUnits = "FOOT_US" ## ### if contUnits == "METER": ### if rasterUnits == "METER": ### outDivide1 = Functions.Divide(divide1_name, 1.0) ### elif rasterUnits == "FOOT_US": ### outDivide1 = Functions.Times(divide1_name, 1200.0 / 3937.0) ### elif rasterUnits == "FOOT_INTL": ### outDivide1 = Functions.Times(divide1_name, 0.3048) ### elif contUnits == "FOOT_US": #### if rasterUnits == "METER": #### outDivide1 = Functions.Times(divide1_name, 1.0 / (1200.0 / 3937.0)) #### elif rasterUnits == "FOOT_US": #### outDivide1 = Functions.Divide(divide1_name, 1.0) #### elif rasterUnits == "FOOT_INTL": #### outDivide1 = Functions.Times(divide1_name, 0.3048 / (1200.0 / 3937.0)) ### elif contUnits == "FOOT_INTL": ### if rasterUnits == "METER": ### outDivide1 = Functions.Times(divide1_name, 1.0 / (0.3048)) ### elif rasterUnits == "FOOT_US": ### outDivide1 = Functions.Times(divide1_name, (1200.0 / 3937.0) / 0.3048) ### elif rasterUnits == "FOOT_INTL": ### outDivide1 = Functions.Divide(divide1_name, 1.0) #### else: #### arcpy.AddMessage("\ncontourUnits: {}, rasterUnits: {}".format(contUnits, rasterUnits)) #### arcpy.AddError('\nUnable to create contours.') #### raise Exception("Units not valid") ## ## outDivide1.save(divide1_path) ## del outDivide1 ## a = doTime(a, '\t' + name + ' ' + index + ': Converted raster units ' + rasterUnits + ' to ' + contUnits + ' = ' + divide1_path) ## ## focal1_name = 'O02_Focal1_' + name + '.tif' ## focal1_path = os.path.join(workspace, focal1_name) ## if not os.path.exists(focal1_path): ## arcpy.MakeRasterLayer_management(in_raster=divide1_path, out_rasterlayer=focal1_name) ## outFS = Functions.FocalStatistics(in_raster=focal1_name, neighborhood="Rectangle 3 3 CELL", statistics_type="MEAN", ignore_nodata="DATA") ## outFS.save(focal1_path) ## del outFS ## a = doTime(a, '\t' + name + ' ' + index + ': Focal statistics on ' + focal1_path) ## ## times1_name = 'O03_Times_' + name + '.tif' ## times1_path = os.path.join(workspace, times1_name) ## if not os.path.exists(times1_path): ## arcpy.MakeRasterLayer_management(in_raster=focal1_path, out_rasterlayer=times1_name) ## outTimes = Functions.Times(times1_name, 100) ## outTimes.save(times1_path) ## del outTimes ## a = doTime(a, '\t' + name + ' ' + index + ': Times 100 on ' + times1_path) ## ## plus1_name = 'O04_Plus_' + name + '.tif' ## plus1_path = os.path.join(workspace, plus1_name) ## if not os.path.exists(plus1_path): ## arcpy.MakeRasterLayer_management(in_raster=times1_path, out_rasterlayer=plus1_name) ## outPlus = Functions.Plus(plus1_name, 0.5) ## outPlus.save(plus1_path) ## del outPlus ## a = doTime(a, '\t' + name + ' ' + index + ': Plus 0.5 ' + plus1_path) ## ## round1_name = 'O05_Round_' + name + '.tif' ## round1_path = os.path.join(workspace, round1_name) ## if not os.path.exists(round1_path): ## arcpy.MakeRasterLayer_management(in_raster=plus1_path, out_rasterlayer=round1_name) ## outRoundDown = Functions.RoundDown(round1_name) ## outRoundDown.save(round1_path) ## del outRoundDown ## a = doTime(a, '\t' + name + ' ' + index + ': Round Down ' + round1_path) ## ## divide2_name = 'O06_Divide2_' + name + '.tif' ## divide2_path = os.path.join(workspace, divide2_name) ## if not os.path.exists(divide2_path): ## arcpy.MakeRasterLayer_management(in_raster=round1_path, out_rasterlayer=divide2_name) ## outDivide2 = Functions.Divide(divide2_name, 100) ## outDivide2.save(divide2_path) ## del outDivide2 ## a = doTime(a, '\t' + name + ' ' + index + ': Divide 100 ' + divide2_path) ## ## focal2_name = 'O07_Focal2_' + name + '.tif' ## focal2_path = os.path.join(workspace, focal2_name) ## if not os.path.exists(focal2_path): ## arcpy.MakeRasterLayer_management(in_raster=divide2_path, out_rasterlayer=focal2_name) ## outFS2 = Functions.FocalStatistics(focal2_name, "Rectangle 3 3 CELL", "MEAN", "DATA") ## outFS2.save(focal2_path) ## del outFS2 ## a = doTime(a, '\t' + name + ' ' + index + ': Focal Statistics #2 ' + focal2_path) # a = doTime(a, '\t{}: Calculating statistics {}'.format(raster_name, contour_ready_path)) # arcpy.CalculateStatistics_management(in_raster_dataset=contour_ready_path, x_skip_factor="1", y_skip_factor="1", ignore_values="", skip_existing="OVERWRITE", area_of_interest="Feature Set") arcpy.AddMessage("\t{}: Referenced Mosaic found '{}'".format(name, focal2_path)) base_name = 'O08_BaseCont_' + name + '.shp' base_contours = os.path.join(workspace, base_name) if not os.path.exists(base_contours): arcpy.MakeRasterLayer_management(in_raster=focal2_path, out_rasterlayer=base_name) Functions.Contour( base_name, base_contours, int(cont_int) ) a = doTime(a, '\t' + name + ' ' + index + ': Contoured to ' + base_contours) simple_contours = os.path.join(workspace, 'O09_SimpleCont_' + name + '.shp') if not os.path.exists(simple_contours): ca.SimplifyLine( base_contours, simple_contours, "POINT_REMOVE", "0.000001 DecimalDegrees", "FLAG_ERRORS", "NO_KEEP", "NO_CHECK" ) a = doTime(a, '\t' + name + ' ' + index + ': Simplified to ' + simple_contours) smooth_contours = os.path.join(workspace, 'O10_SmoothCont_' + name + '.shp') if not os.path.exists(smooth_contours): ca.SmoothLine( simple_contours, smooth_contours, "PAEK", "{} DecimalDegrees".format(smooth_tol), "", "NO_CHECK" ) a = doTime(a, '\t' + name + ' ' + index + ': Smoothed to ' + smooth_contours) # put this up one level to avoid re-processing all of above if something goes wrong below clip_workspace = os.path.split(workspace)[0] clip_contours = os.path.join(clip_workspace, 'O11_ClipCont_' + name + '.shp') if not os.path.exists(clip_contours): arcpy.Clip_analysis( in_features=smooth_contours, clip_features=clip_poly, out_feature_class=clip_contours ) a = doTime(a, '\t' + name + ' ' + index + ': Clipped to ' + clip_contours) arcpy.RepairGeometry_management(in_features=clip_contours, delete_null="DELETE_NULL") Utility.addAndCalcFieldLong(dataset_path=clip_contours, field_name="CTYPE", field_value="getType( !CONTOUR! )", code_block="def getType(contour):\n\n type = 2\n\n if contour%10 == 0:\n\n type = 10\n\n if contour%20 == 0:\n\n type = 20\n\n if contour%50 == 0:\n type = 50\n if contour%100 == 0:\n type = 100\n if contour%500 == 0:\n type = 500\n if contour%1000 == 0:\n type = 1000\n if contour%5000 == 0:\n type = 5000\n return type", add_index=False) Utility.addAndCalcFieldLong(dataset_path=clip_contours, field_name="INDEX", field_value="getType( !CONTOUR! )", code_block="def getType(contour):\n\n type = 0\n\n if contour%" + str(int(cont_int * 5)) + " == 0:\n\n type = 1\n return type", add_index=False) # Utility.addAndCalcFieldText(dataset_path=clip_contours, # field_name="LastMergedFC", # field_length=100, # field_value=name, # add_index=False) # Utility.addAndCalcFieldText(dataset_path=clip_contours, # field_name="ValidationCheck", # field_length=100, # field_value='"'+name+'"', # add_index=False) Utility.addAndCalcFieldText(dataset_path=clip_contours, field_name="UNITS", field_length=20, field_value='"' + CONTOUR_UNIT + '"', add_index=False) Utility.addAndCalcFieldText(dataset_path=clip_contours, field_name="name", field_length=79, field_value='"' + name + '"', add_index=False) a = doTime(a, '\t' + name + ' ' + index + ': Added fields to ' + clip_contours) try: arcpy.DeleteField_management(in_table=clip_contours, drop_field="ID;InLine_FID;SimLnFlag;MaxSimpTol;MinSimpTol") a = doTime(a, '\t' + name + ' ' + index + ': Deleted fields from ' + clip_contours) except: pass doTime(aa, 'FINISHED ' + name + ' ' + index) created = True except Exception as e: arcpy.AddMessage('\t\tProcess Dropped: ' + name) arcpy.AddMessage('\t\tException: ' + str(e)) if tries > TRIES_ALLOWED: arcpy.AddError('Too many tries, Dropped: {}'.format(name)) try: arcpy.AddMessage("Checking in licenses") arcpy.CheckInExtension("3D") arcpy.CheckInExtension("Spatial") except: pass