def calc_lccs(normalize): try: if normalize: mosaicBaseName = "_corridors" writeTruncRaster = cfg.WRITETRUNCRASTER outputGDB = cfg.OUTPUTGDB SAVENORMLCCS = cfg.SAVENORMLCCS else: mosaicBaseName = "_NON_NORMALIZED_corridors" SAVENORMLCCS = False outputGDB = cfg.EXTRAGDB writeTruncRaster = False lu.dashline(1) gprint('Running script ' + _SCRIPT_NAME) linkTableFile = lu.get_prev_step_link_table(step=5) arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.compression = "NONE" if cfg.MAXEUCDIST is not None: gprint('Max Euclidean distance between cores') gprint('for linkage mapping set to ' + str(cfg.MAXEUCDIST)) if cfg.MAXCOSTDIST is not None: gprint('Max cost-weighted distance between cores') gprint('for linkage mapping set to ' + str(cfg.MAXCOSTDIST)) # set the analysis extent and cell size to that of the resistance # surface arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = arcpy.Describe(cfg.RESRAST).MeanCellHeight arcpy.env.snapRaster = cfg.RESRAST arcpy.env.mask = cfg.RESRAST linkTable = lu.load_link_table(linkTableFile) numLinks = linkTable.shape[0] numCorridorLinks = lu.report_links(linkTable) if numCorridorLinks == 0: lu.dashline(1) msg =('\nThere are no corridors to map. Bailing.') lu.raise_error(msg) if not cfg.STEP3 and not cfg.STEP4: # re-check for links that are too long or in case script run out of # sequence with more stringent settings gprint('Double-checking for corridors that are too long to map.') DISABLE_LEAST_COST_NO_VAL = True linkTable,numDroppedLinks = lu.drop_links( linkTable, cfg.MAXEUCDIST, cfg.MINEUCDIST, cfg.MAXCOSTDIST, cfg.MINCOSTDIST, DISABLE_LEAST_COST_NO_VAL) # Added to try to speed up: arcpy.env.pyramid = "NONE" arcpy.env.rasterStatistics = "NONE" # set up directories for normalized lcc and mosaic grids dirCount = 0 gprint("Creating output folder: " + cfg.LCCBASEDIR) lu.delete_dir(cfg.LCCBASEDIR) arcpy.CreateFolder_management(path.dirname(cfg.LCCBASEDIR), path.basename(cfg.LCCBASEDIR)) arcpy.CreateFolder_management(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM) clccdir = path.join(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM) gprint("") if normalize: gprint('Normalized least-cost corridors will be written ' 'to ' + clccdir + '\n') PREFIX = cfg.PREFIX # Add CWD layers for core area pairs to produce NORMALIZED LCC layers numGridsWritten = 0 coreList = linkTable[:,cfg.LTB_CORE1:cfg.LTB_CORE2+1] coreList = npy.sort(coreList) x = 0 linkCount = 0 endIndex = numLinks while x < endIndex: if (linkTable[x, cfg.LTB_LINKTYPE] < 1): # If not a valid link x = x + 1 continue linkCount = linkCount + 1 start_time = time.clock() linkId = str(int(linkTable[x, cfg.LTB_LINKID])) # source and target cores corex=int(coreList[x,0]) corey=int(coreList[x,1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) if not arcpy.Exists(cwdRaster1): msg =('\nError: cannot find cwd raster:\n' + cwdRaster1) if not arcpy.Exists(cwdRaster2): msg =('\nError: cannot find cwd raster:\n' + cwdRaster2) lu.raise_error(msg) lccNormRaster = path.join(clccdir, str(corex) + "_" + str(corey))# + ".tif") arcpy.env.extent = "MINOF" link = lu.get_links_from_core_pairs(linkTable, corex, corey) offset = 10000 # Normalized lcc rasters are created by adding cwd rasters and # subtracting the least cost distance between them. lcDist = (float(linkTable[link,cfg.LTB_CWDIST]) - offset) if normalize: statement = ('outras = arcpy.sa.Raster(cwdRaster1) ' '+ arcpy.sa.Raster(cwdRaster2) - lcDist; ' 'outras.save(lccNormRaster)') else: statement = ('outras = arcpy.sa.Raster(cwdRaster1) ' '+ arcpy.sa.Raster(cwdRaster2); ' 'outras.save(lccNormRaster)') count = 0 while True: try: exec(statement) except Exception: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec(statement) else: break if normalize: try: minObject = arcpy.GetRasterProperties_management(lccNormRaster, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) except Exception: lu.warn('\n------------------------------------------------') lu.warn('WARNING: Raster minimum check failed in step 5. \n' 'This may mean the output rasters are corrupted. Please \n' 'be sure to check for valid rasters in '+ outputGDB) rasterMin = 0 tolerance = (float(arcpy.env.cellSize) * -10) if rasterMin < tolerance: lu.dashline(1) msg = ('WARNING: Minimum value of a corridor #' + str(x+1) + ' is much less than zero ('+str(rasterMin)+').' '\nThis could mean that BOUNDING CIRCLE BUFFER DISTANCES ' 'were too small and a corridor passed outside of a ' 'bounding circle, or that a corridor passed outside of the ' 'resistance map. \n') lu.warn(msg) arcpy.env.extent = cfg.RESRAST mosaicDir = path.join(cfg.LCCBASEDIR,'mos'+str(x+1)) lu.create_dir(mosaicDir) mosFN = 'mos'#.tif' change and move mosaicRaster = path.join(mosaicDir,mosFN) if numGridsWritten == 0 and dirCount == 0: #If this is the first grid then copy rather than mosaic arcpy.CopyRaster_management(lccNormRaster, mosaicRaster) else: statement = ( 'arcpy.MosaicToNewRaster_management(' 'input_rasters=";".join([lccNormRaster, ' 'lastMosaicRaster]), output_location=mosaicDir, ' 'raster_dataset_name_with_extension=mosFN, ' 'pixel_type="32_BIT_FLOAT", cellsize=arcpy.env.cellSize, ' 'number_of_bands="1", mosaic_method="MINIMUM")') count = 0 while True: try: lu.write_log('Executing mosaic for link #'+str(linkId)) exec(statement) lu.write_log('Done with mosaic.') except Exception: count,tryAgain = lu.retry_arc_error(count,statement) lu.delete_data(mosaicRaster) lu.delete_dir(mosaicDir) # Try a new directory mosaicDir = path.join(cfg.LCCBASEDIR,'mos'+str(x+1)+ '_' + str(count)) lu.create_dir(mosaicDir) mosaicRaster = path.join(mosaicDir,mosFN) if not tryAgain: exec(statement) else: break endTime = time.clock() processTime = round((endTime - start_time), 2) if normalize == True: printText = "Normalized and mosaicked " else: printText = "Mosaicked NON-normalized " gprint(printText + "corridor for link ID #" + str(linkId) + " connecting core areas " + str(corex) + " and " + str(corey)+ " in " + str(processTime) + " seconds. " + str(int(linkCount)) + " out of " + str(int(numCorridorLinks)) + " links have been " "processed.") # temporarily disable links in linktable - don't want to mosaic # them twice for y in range (x+1,numLinks): corex1 = int(coreList[y,0]) corey1 = int(coreList[y,1]) if corex1 == corex and corey1 == corey: linkTable[y,cfg.LTB_LINKTYPE] = ( linkTable[y,cfg.LTB_LINKTYPE] + 1000) elif corex1==corey and corey1==corex: linkTable[y,cfg.LTB_LINKTYPE] = ( linkTable[y,cfg.LTB_LINKTYPE] + 1000) numGridsWritten = numGridsWritten + 1 if not SAVENORMLCCS: lu.delete_data(lccNormRaster) lu.delete_dir(clccdir) lu.create_dir(clccdir) else: if numGridsWritten == 100: # We only write up to 100 grids to any one folder # because otherwise Arc slows to a crawl dirCount = dirCount + 1 numGridsWritten = 0 clccdir = path.join(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM + str(dirCount)) gprint("Creating output folder: " + clccdir) arcpy.CreateFolder_management(cfg.LCCBASEDIR, path.basename(clccdir)) if numGridsWritten > 1 or dirCount > 0: lu.delete_data(lastMosaicRaster) lu.delete_dir(path.dirname(lastMosaicRaster)) lastMosaicRaster = mosaicRaster x = x + 1 #rows that were temporarily disabled rows = npy.where(linkTable[:,cfg.LTB_LINKTYPE]>1000) linkTable[rows,cfg.LTB_LINKTYPE] = ( linkTable[rows,cfg.LTB_LINKTYPE] - 1000) # --------------------------------------------------------------------- # Create output geodatabase if not arcpy.Exists(outputGDB): arcpy.CreateFileGDB_management(cfg.OUTPUTDIR, path.basename(outputGDB)) arcpy.env.workspace = outputGDB arcpy.env.pyramid = "NONE" arcpy.env.rasterStatistics = "NONE" # --------------------------------------------------------------------- # convert mosaic raster to integer intRaster = path.join(outputGDB,PREFIX + mosaicBaseName) statement = ('outras = arcpy.sa.Int(arcpy.sa.Raster(mosaicRaster) ' '- offset + 0.5); ' 'outras.save(intRaster)') count = 0 while True: try: exec(statement) except Exception: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec(statement) else: break # --------------------------------------------------------------------- if writeTruncRaster: # ----------------------------------------------------------------- # Set anything beyond cfg.CWDTHRESH to NODATA. truncRaster = (outputGDB + '\\' + PREFIX + mosaicBaseName + '_truncated_at_' + lu.cwd_cutoff_str(cfg.CWDTHRESH)) statement = ('outRas = arcpy.sa.Raster(intRaster)' '* (arcpy.sa.Con(arcpy.sa.Raster(intRaster) ' '<= cfg.CWDTHRESH, 1)); ' 'outRas.save(truncRaster)') count = 0 while True: try: exec(statement) except Exception: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec(statement) else: break # --------------------------------------------------------------------- # Check for unreasonably low minimum NLCC values try: mosaicGrid = path.join(cfg.LCCBASEDIR,'mos') # Copy to grid to test arcpy.CopyRaster_management(mosaicRaster, mosaicGrid) minObject = arcpy.GetRasterProperties_management(mosaicGrid, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) except Exception: lu.warn('\n------------------------------------------------') lu.warn('WARNING: Raster minimum check failed in step 5. \n' 'This may mean the output rasters are corrupted. Please \n' 'be sure to check for valid rasters in '+ outputGDB) rasterMin = 0 tolerance = (float(arcpy.env.cellSize) * -10) if rasterMin < tolerance: lu.dashline(1) msg = ('WARNING: Minimum value of mosaicked corridor map is ' 'much less than zero ('+str(rasterMin)+').' '\nThis could mean that BOUNDING CIRCLE BUFFER DISTANCES ' 'were too small and a corridor passed outside of a ' 'bounding circle, or that a corridor passed outside of the ' 'resistance map. \n') lu.warn(msg) gprint('\nWriting final LCP maps...') if cfg.STEP4: finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=4, thisStep=5) elif cfg.STEP3: finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=3, thisStep=5) else: # Don't know if step 4 was run, since this is started at step 5. # Use presence of previous linktable files to figure this out. # Linktable name includes step number. prevLinkTableFile = lu.get_prev_step_link_table(step=5) prevStepInd = len(prevLinkTableFile) - 5 lastStep = prevLinkTableFile[prevStepInd] finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep, thisStep=5) outlinkTableFile = lu.get_this_step_link_table(step=5) gprint('Updating ' + outlinkTableFile) lu.write_link_table(linkTable, outlinkTableFile) linkTableLogFile = path.join(cfg.LOGDIR, "linkTable_s5.csv") lu.write_link_table(linkTable, linkTableLogFile) linkTableFinalFile = path.join(cfg.OUTPUTDIR, PREFIX + "_linkTable_s5.csv") lu.write_link_table(finalLinkTable, linkTableFinalFile) gprint('Copy of final linkTable written to '+ linkTableFinalFile) gprint('Creating shapefiles with linework for links.') try: lu.write_link_maps(outlinkTableFile, step=5) except Exception: lu.write_link_maps(outlinkTableFile, step=5) # Create final linkmap files in output directory, and remove files from # scratch. lu.copy_final_link_maps(step=5) if not SAVENORMLCCS: lu.delete_dir(cfg.LCCBASEDIR) # Build statistics for corridor rasters arcpy.AddMessage('\nBuilding output statistics and pyramids ' 'for corridor raster') lu.build_stats(intRaster) if writeTruncRaster: arcpy.AddMessage('Building output statistics ' 'for truncated corridor raster') lu.build_stats(truncRaster) save_parameters() if cfg.OUTPUTFORMODELBUILDER: arcpy.CopyFeatures_management(cfg.COREFC, cfg.OUTPUTFORMODELBUILDER) # Return GEOPROCESSING specific errors except arcpy.ExecuteError: lu.dashline(1) gprint('****Failed in step 5. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except Exception: lu.dashline(1) gprint('****Failed in step 5. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME) return
def do_cwd_calcs(x, linkTable, coresToMap, lcpLoop, failures): try: # This is the focal core area we're running cwd out from sourceCore = int(coresToMap[x]) # Create temporary scratch directory just this focal core coreDir = path.join(cfg.SCRATCHDIR, 'core' + str(sourceCore)) lu.delete_dir(coreDir) lu.create_dir(coreDir) if arcpy: gp = arcpy.gp arcpy.env.workspace = coreDir arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.overwriteOutput = True arcpy.env.extent = "MINOF" else: gp = cfg.gp gp.workspace = coreDir gp.scratchWorkspace = cfg.ARCSCRATCHDIR gp.OverwriteOutput = True gp.Extent = "MINOF" write_cores_to_map(x, coresToMap) # Get target cores based on linktable with reinstated links # (we temporarily disable them below by adding 1000) linkTableTemp = linkTable.copy() # reinstate temporarily disabled links rows = npy.where(linkTableTemp[:,cfg.LTB_LINKTYPE] > 1000) linkTableTemp[rows,cfg.LTB_LINKTYPE] = ( linkTableTemp[rows,cfg.LTB_LINKTYPE] - 1000) del rows # get core areas to be connected to focal core targetCores = lu.get_core_targets(sourceCore, linkTableTemp) # gprint( str(sourceCore)) # gprint(str(linkTableTemp.astype('int32'))) # gprint('targets'+str(targetCores)) del linkTableTemp if len(targetCores)==0: # Nothing to do, so reset failure count and return. failures = 0 return linkTable, failures, lcpLoop lu.dashline(0) gprint('Target core areas for core area #' + str(sourceCore) + ' = ' + str(targetCores)) # ------------------------------------------------------------- # Create BOUNDING FEATURE to limit extent of cost distance # calculations-This is a set of circles encompassing core areas # we'll be connecting each core area to. if cfg.BUFFERDIST is not None: # fixme: move outside of loop # new circle gp.MakeFeatureLayer(cfg.BNDCIRS,"fGlobalBoundingFeat") start_time = time.clock() # loop through targets and get bounding circles that # contain focal core and target cores # gprint("\nAdding up bounding circles for source" # " core " + str(sourceCore)) gp.SelectLayerByAttribute("fGlobalBoundingFeat", "CLEAR_SELECTION") for i in range(len(targetCores)): # run thru circleList, find link that core pair # corresponds to. if sourceCore < targetCores[i]: corex = sourceCore corey = targetCores[i] else: corey = sourceCore corex = targetCores[i] cores_x_y = str(int(corex))+'_'+str(int(corey)) field = "cores_x_y" # fixme: need to check for case where link is not found gp.SelectLayerByAttribute( "fGlobalBoundingFeat", "ADD_TO_SELECTION", field + " = '" + cores_x_y + "'") lu.delete_data(path.join(coreDir,cfg.BNDFC)) # fixme: may not be needed- can we just clip raster # using selected? gp.CopyFeatures_management("fGlobalBoundingFeat", cfg.BNDFC) # Clip out bounded area of resistance raster for cwd # calculations from focal core bResistance = path.join(coreDir,"bResistance") # Can't be tif- # need STA for CWD lu.delete_data(bResistance) statement = ( 'gp.ExtractByMask_sa(cfg.BOUNDRESIS, cfg.BNDFC, bResistance)') try: exec statement randomerror() except: failures = lu.print_arcgis_failures(statement, failures) if failures < 20: return None,failures,lcpLoop else: exec statement else: bResistance = cfg.BOUNDRESIS # --------------------------------------------------------- # CWD Calculations outDistanceRaster = lu.get_cwd_path(sourceCore) # Check if climate tool is calling linkage mapper if cfg.TOOL == cfg.TOOL_CC: back_rast = outDistanceRaster.replace("cwd_", "back_") else: back_rast = "BACK" lu.delete_data(path.join(coreDir, back_rast)) lu.delete_data(outDistanceRaster) start_time = time.clock() # Create raster that just has source core in it # Note: this seems faster than setnull with LI grid. SRCRASTER = 'source' + tif lu.delete_data(path.join(coreDir,SRCRASTER)) if arcpy: statement = ('conRaster = ' 'Con(Raster(cfg.CORERAS) == int(sourceCore), 1);' 'conRaster.save(SRCRASTER)') else: expression = ("con(" + cfg.CORERAS + " == " + str(int(sourceCore)) + ", 1)") statement = ('gp.SingleOutputMapAlgebra_sa' '(expression, SRCRASTER)') try: exec statement randomerror() except: failures = lu.print_arcgis_failures(statement, failures) if failures < 20: return None, failures, lcpLoop else: exec statement # Cost distance raster creation if arcpy: arcpy.env.extent = "MINOF" else: gp.Extent = "MINOF" lu.delete_data(path.join(coreDir,"BACK")) if arcpy: statement = ('outCostDist = CostDistance(SRCRASTER, ' 'bResistance, cfg.TMAXCWDIST, back_rast);' 'outCostDist.save(outDistanceRaster)') else: statement = ('gp.CostDistance_sa(SRCRASTER, bResistance, ' 'outDistanceRaster, cfg.TMAXCWDIST, back_rast)') try: exec statement randomerror() except: failures = lu.print_arcgis_failures(statement, failures) if failures < 20: return None, failures, lcpLoop else: exec statement start_time = time.clock() # Extract cost distances from source core to target cores # Fixme: there will be redundant calls to b-a when already # done a-b ZNSTATS = path.join(coreDir, "zonestats.dbf") lu.delete_data(ZNSTATS) #Fixme: zonalstatistics is returning integer values for minimum. Why??? #Extra zonalstatistics code implemented later in script to correct #values. if arcpy: statement = ('outZSaT = ZonalStatisticsAsTable(cfg.CORERAS, ' '"VALUE", outDistanceRaster,ZNSTATS, "DATA", "MINIMUM")') else: statement = ('gp.zonalstatisticsastable_sa(' 'cfg.CORERAS, "VALUE", outDistanceRaster, ZNSTATS)') try: exec statement randomerror() except: failures = lu.print_arcgis_failures(statement, failures) if failures < 20: return None,failures,lcpLoop else: if cfg.TOOL == cfg.TOOL_CC: msg = ('ERROR in Zonal Stats. Please restart ArcMap ' 'and try again.') else: msg = ('ERROR in Zonal Stats. Restarting ArcMap ' 'then restarting Linkage Mapper at step 3 usually\n' 'solves this one so please restart and try again.') lu.raise_error(msg) tableRows = gp.searchcursor(ZNSTATS) tableRow = tableRows.Next() while tableRow: if tableRow.Value > sourceCore: link = lu.get_links_from_core_pairs(linkTable, sourceCore, tableRow.Value) if linkTable[link,cfg.LTB_LINKTYPE] > 0: # valid link linkTable[link,cfg.LTB_CWDIST] = tableRow.Min if cfg.MAXCOSTDIST is not None: if ((tableRow.Min > cfg.MAXCOSTDIST) and (linkTable[link,cfg.LTB_LINKTYPE] != cfg.LT_KEEP)): # Disable link, it's too long linkTable[link,cfg.LTB_LINKTYPE] = cfg.LT_TLLC if cfg.MINCOSTDIST is not None: if (tableRow.Min < cfg.MINCOSTDIST and (linkTable[link,cfg.LTB_LINKTYPE] != cfg.LT_KEEP)): # Disable link, it's too short linkTable[link,cfg.LTB_LINKTYPE] = cfg.LT_TSLC tableRow = tableRows.next() del tableRow, tableRows #start_time = lu.elapsed_time(start_time) # --------------------------------------------------------- # Check for intermediate cores AND map LCP lines for y in range(0,len(targetCores)): targetCore = targetCores[y] rows = lu.get_links_from_core_pairs(linkTable, sourceCore, targetCore) # Map all links for which we successfully extracted # cwds in above code link = rows[0] if (linkTable[link,cfg.LTB_LINKTYPE] > 0 and linkTable[link,cfg.LTB_LINKTYPE] < 1000 and linkTable[link,cfg.LTB_CWDIST] != -1): # Flag so that we only evaluate this pair once linkTable[rows,cfg.LTB_LINKTYPE] = (linkTable [rows,cfg.LTB_LINKTYPE] + 1000) # Create raster that just has target core in it TARGETRASTER = 'targ' + tif lu.delete_data(path.join(coreDir,TARGETRASTER)) try: if arcpy: # For climate corridors, errors occur when core raster # overlaps null values in cwd rasters statement = ('conRaster = Con(IsNull(outDistanceRaster' '), Int(outDistanceRaster), Con(Raster' '(cfg.CORERAS) == int(targetCore), 1));' 'conRaster.save(TARGETRASTER)') # statement = ('conRaster = Con(Raster(' # 'cfg.CORERAS) == int(targetCore), 1);' # 'conRaster.save(TARGETRASTER)') else: expression = ("con(" + cfg.CORERAS + " == " + str(int(targetCore)) + ",1)") statement = ('gp.SingleOutputMapAlgebra_sa(expression,' ' TARGETRASTER)') exec statement randomerror() except: failures = lu.print_arcgis_failures(statement, failures) if failures < 20: return None,failures,lcpLoop else: exec statement # Execute ZonalStatistics to get more precise cw distance if # arc rounded it earlier (not critical, hence the try/pass) if (linkTable[link,cfg.LTB_CWDIST] == int(linkTable[link,cfg.LTB_CWDIST])): try: zonalRas = path.join(coreDir,'zonal') gp.ZonalStatistics_sa(TARGETRASTER, "VALUE", outDistanceRaster, zonalRas, "MINIMUM", "DATA") minObject = gp.GetRasterProperties_management(zonalRas, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) linkTable[link,cfg.LTB_CWDIST] = rasterMin lu.delete_data(zonalRas) except: pass # Cost path maps the least cost path # between source and target lcpRas = path.join(coreDir,"lcp" + tif) lu.delete_data(lcpRas) # Note: costpath (both gp and arcpy versions) uses GDAL. if arcpy: statement = ('outCostPath = CostPath(TARGETRASTER,' 'outDistanceRaster, back_rast, "BEST_SINGLE", ""); ' 'outCostPath.save(lcpRas)') else: statement = ('gp.CostPath_sa(TARGETRASTER, ' 'outDistanceRaster, back_rast, ' 'lcpRas, "BEST_SINGLE", "")') try: exec statement randomerror() except: failures = lu.print_arcgis_failures(statement, failures) if failures < 20: return None,failures,lcpLoop else: lu.dashline(1) gprint('\nCost path is failing for Link #' + str(int(link)) + ' connecting core areas ' + str(int(sourceCore)) + ' and ' + str(int(targetCore)) + '\n.' 'Retrying one more time in 5 minutes.') lu.snooze(300) exec statement # fixme: may be fastest to not do selection, do # EXTRACTBYMASK, getvaluelist, use code snippet at end # of file to discard src and target values. Still this # is fast- 13 sec for LI data...But I'm not very # comfortable using failed coreMin as our test.... if (cfg.S3DROPLCCSic and (linkTable[link,cfg.LTB_LINKTYPE] != cfg.LT_KEEP) and (linkTable[link,cfg.LTB_LINKTYPE] != cfg.LT_KEEP + 1000)): # ------------------------------------------------- # Drop links where lcp passes through intermediate # core area. Method below is faster than valuelist # method because of soma in valuelist method. # make a feature layer for input cores to select from gp.MakeFeatureLayer(cfg.COREFC, cfg.FCORES) gp.SelectLayerByAttribute(cfg.FCORES, "NEW_SELECTION", cfg.COREFN + ' <> ' + str(int(targetCore)) + ' AND ' + cfg.COREFN + ' <> ' + str(int(sourceCore))) corePairRas = path.join(coreDir,"s3corepair"+ tif) if arcpy: arcpy.env.extent = cfg.BOUNDRESIS else: gp.extent = gp.Describe(cfg.BOUNDRESIS).extent statement = ('gp.FeatureToRaster_conversion(cfg.FCORES, ' 'cfg.COREFN, corePairRas, gp.cellSize)') try: exec statement randomerror() except: failures = lu.print_arcgis_failures(statement, failures) if failures < 20: return None,failures,lcpLoop else: exec statement #------------------------------------------ # Intermediate core test try: coreDetected = test_for_intermediate_core(coreDir, lcpRas, corePairRas) randomerror() except: statement = 'test_for_intermediate_core' failures = lu.print_arcgis_failures(statement, failures) if failures < 20: return None,failures,lcpLoop else: coreDetected = test_for_intermediate_core( coreDir, lcpRas, corePairRas) if coreDetected: # lu.dashline() gprint( "Found an intermediate core in the " "least-cost path between cores " + str(int(sourceCore)) + " and " + str(int(targetCore)) + ". The corridor " "will be removed.") # disable link rows = lu.get_links_from_core_pairs(linkTable, sourceCore, targetCore) linkTable[rows,cfg.LTB_LINKTYPE] = cfg.LT_INT #------------------------------------------ # Create lcp shapefile. lcploop just keeps track of # whether this is first time function is called. lcpLoop = lu.create_lcp_shapefile(coreDir, linkTable, sourceCore, targetCore, lcpLoop) # Made it through, so reset failure count and return. failures = 0 lu.delete_dir(coreDir) # if cfg.TOOL == cfg.TOOL_CC: # lu.delete_data(back_rast) return linkTable, failures, lcpLoop # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.dashline(1) # gprint('****Failed in step 3. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) # gprint('****Failed in step 3. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME)
def STEP3_calc_cwds(): """Calculates cost-weighted distances from each core area. Uses bounding circles around source and target cores to limit extent of cwd calculations and speed computation. """ try: lu.dashline(1) gprint('Running script ' + _SCRIPT_NAME) lu.dashline(0) # Super secret setting to re-start failed run. Enter 'RESTART' as the # Name of the pairwise distance table in step 2, and uncheck step 2. # We can eventually place this in a .ini file. rerun = False if cfg.S2EUCDISTFILE != None: if cfg.S2EUCDISTFILE.lower() == "restart": rerun = True # if cfg.TMAXCWDIST is None: # gprint('NOT using a maximum cost-weighted distance.') # else: # gprint('Max cost-weighted distance for CWD calcs set ' # 'to ' + str(cfg.TMAXCWDIST) + '\n') if (cfg.BUFFERDIST) is not None: gprint('Bounding circles plus a buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units will ' 'be used \n to limit extent of cost distance ' 'calculations.') elif cfg.TOOL <> cfg.TOOL_CC: gprint('NOT using bounding circles in cost distance ' 'calculations.') # set the analysis extent and cell size # So we don't extract rasters that go beyond extent of original raster if arcpy: arcpy.env.cellSize = cfg.RESRAST arcpy.env.extent="MINOF" else: gp.cellSize = gp.Describe(cfg.RESRAST).MeanCellHeight gp.Extent = "MINOF" gp.mask = cfg.RESRAST if arcpy: arcpy.env.overwriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR else: gp.OverwriteOutput = True gp.workspace = cfg.SCRATCHDIR gp.scratchWorkspace = cfg.ARCSCRATCHDIR # Load linkTable (created in previous script) linkTableFile = lu.get_prev_step_link_table(step=3) linkTable = lu.load_link_table(linkTableFile) lu.report_links(linkTable) # Identify cores to map from LinkTable coresToMap = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) numCoresToMap = len(coresToMap) if numCoresToMap < 3: # No need to check for intermediate cores, because there aren't any cfg.S3DROPLCCSic = False else: cfg.S3DROPLCCSic = cfg.S3DROPLCCS gprint('\nNumber of core areas to connect: ' + str(numCoresToMap)) if rerun: # If picking up a failed run, make sure needed files are there lu.dashline(1) gprint ('\n****** RESTART MODE ENABLED ******\n') gprint ('**** NOTE: This mode picks up step 3 where a\n' 'previous run left off due to a crash or user\n' 'abort. It assumes you are using the same input\n' 'data used in the terminated run.\n\n') lu.warn('IMPORTANT: Your LCP and stick feature classes\n' 'will LOSE LCPs that were already created, but\n' 'your final raster corridor map should be complete.\n') lu.dashline(0) lu.snooze(10) savedLinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") coreListFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") if not path.exists(savedLinkTableFile) or not path.exists( coreListFile): gprint('No partial results file found from previous ' 'stopped run. Starting run from beginning.\n') lu.dashline(0) rerun = False # If picking up a failed run, use old folders if not rerun: startIndex = 0 if cfg.TOOL <> cfg.TOOL_CC: lu.make_cwd_paths(max(coresToMap)) # Set up cwd directories # make a feature layer for input cores to select from gp.MakeFeatureLayer(cfg.COREFC, cfg.FCORES) # Drop links that are too long gprint('\nChecking for corridors that are too long to map.') DISABLE_LEAST_COST_NO_VAL = False linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, 0, cfg.MAXCOSTDIST, 0, DISABLE_LEAST_COST_NO_VAL) # ------------------------------------------------------------------ # Bounding boxes if (cfg.BUFFERDIST) is not None: # create bounding boxes around cores start_time = time.clock() # lu.dashline(1) gprint('Calculating bounding boxes for core areas.') extentBoxList = npy.zeros((0,5), dtype='float32') for x in range(len(coresToMap)): core = coresToMap[x] boxCoords = lu.get_extent_box_coords(core) extentBoxList = npy.append(extentBoxList, boxCoords, axis=0) gprint('\nDone calculating bounding boxes.') start_time = lu.elapsed_time(start_time) # lu.dashline() # Bounding circle code if cfg.BUFFERDIST is not None: # Make a set of circles encompassing core areas we'll be connecting start_time = time.clock() gprint('Calculating bounding circles around potential' ' corridors.') # x y corex corey radius- stores data for bounding circle centroids boundingCirclePointArray = npy.zeros((0,5), dtype='float32') circleList = npy.zeros((0,3), dtype='int32') numLinks = linkTable.shape[0] for x in range(0, numLinks): if ((linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_CORR) or (linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_KEEP)): # if it's a valid corridor link linkId = int(linkTable[x,cfg.LTB_LINKID]) # fixme- this code is clumsy- can trim down cores = npy.zeros((1,3), dtype='int32') cores[0,:] = npy.sort([0, linkTable[x,cfg.LTB_CORE1], linkTable[x,cfg.LTB_CORE2]]) corex = cores[0,1] corey = cores[0,2] cores[0,0] = linkId ################### foundFlag = False for y in range(0,len(circleList)): # clumsy if (circleList[y,1] == corex and circleList[y,2] == corey): foundFlag = True if not foundFlag: circlePointData = ( lu.get_bounding_circle_data(extentBoxList, corex, corey, cfg.BUFFERDIST)) boundingCirclePointArray = ( npy.append(boundingCirclePointArray, circlePointData, axis=0)) # keep track of which cores we draw bounding circles # around circleList = npy.append(circleList, cores, axis=0) gprint('\nCreating bounding circles using buffer ' 'analysis.') dir, BNDCIRCENS = path.split(cfg.BNDCIRCENS) lu.make_points(cfg.SCRATCHDIR, boundingCirclePointArray, BNDCIRCENS) lu.delete_data(cfg.BNDCIRS) gp.buffer_analysis(cfg.BNDCIRCENS, cfg.BNDCIRS, "radius") gp.deletefield (cfg.BNDCIRS, "BUFF_DIST") gprint('Successfully created bounding circles around ' 'potential corridors using \na buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.') start_time = lu.elapsed_time(start_time) gprint('Reducing global processing area using bounding ' 'circle plus buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.\n') extentBoxList = npy.zeros((0,5),dtype='float32') boxCoords = lu.get_extent_box_coords() extentBoxList = npy.append(extentBoxList,boxCoords,axis=0) extentBoxList[0,0] = 0 boundingCirclePointArray = npy.zeros((0,5),dtype='float32') circlePointData=lu.get_bounding_circle_data(extentBoxList, 0, 0, cfg.BUFFERDIST) dir, BNDCIRCEN = path.split(cfg.BNDCIRCEN) lu.make_points(cfg.SCRATCHDIR, circlePointData, BNDCIRCEN) lu.delete_data(cfg.BNDCIR) gp.buffer_analysis(cfg.BNDCIRCEN, cfg.BNDCIR, "radius") gprint('Extracting raster....') cfg.BOUNDRESIS = cfg.BOUNDRESIS + tif lu.delete_data(cfg.BOUNDRESIS) count = 0 statement = ( 'gp.ExtractByMask_sa(cfg.RESRAST, cfg.BNDCIR, cfg.BOUNDRESIS)') while True: try: exec statement randomerror() except: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec statement else: break gprint('\nReduced resistance raster extracted using ' 'bounding circle.') else: #if not using bounding circles, just go with resistance raster. cfg.BOUNDRESIS = cfg.RESRAST # --------------------------------------------------------------------- # Rasterize core areas to speed cost distance calcs # lu.dashline(1) gprint("Creating core area raster.") gp.SelectLayerByAttribute(cfg.FCORES, "CLEAR_SELECTION") if arcpy: arcpy.env.cellSize = cfg.BOUNDRESIS arcpy.env.extent = cfg.BOUNDRESIS else: gp.cellSize = gp.Describe(cfg.BOUNDRESIS).MeanCellHeight gp.extent = gp.Describe(cfg.BOUNDRESIS).extent if rerun: # saved linktable replaces the one now in memory linkTable = lu.load_link_table(savedLinkTableFile) coresToMapSaved = npy.loadtxt(coreListFile, dtype='Float64', comments='#', delimiter=',') startIndex = coresToMapSaved[0] # Index of core where we left off del coresToMapSaved gprint ('\n****** Re-starting run at core area number ' + str(int(coresToMap[startIndex]))+ ' ******\n') lu.dashline(0) if arcpy: arcpy.env.extent = "MINOF" else: gp.extent = "MINOF" #---------------------------------------------------------------------- # Loop through cores, do cwd calcs for each if cfg.TOOL == cfg.TOOL_CC: gprint("\nMapping least-cost paths.\n") else: gprint("\nStarting cost distance calculations.\n") lcpLoop = 0 failures = 0 x = startIndex endIndex = len(coresToMap) linkTableMod = linkTable.copy() while x < endIndex: startTime1 = time.clock() # Modification of linkTable in function was causing problems. so # make a copy: linkTablePassed = linkTableMod.copy() (linkTableReturned, failures, lcpLoop) = do_cwd_calcs(x, linkTablePassed, coresToMap, lcpLoop, failures) if failures == 0: # If iteration was successful, continue with next core linkTableMod = linkTableReturned sourceCore = int(coresToMap[x]) gprint('Done with all calculations for core ID #' + str(sourceCore) + '. ' + str(int(x + 1)) + ' of ' + str(endIndex) + ' cores have been processed.') start_time = lu.elapsed_time(startTime1) outlinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.write_link_table(linkTableMod, outlinkTableFile) # Increment loop counter x = x + 1 else: # If iteration failed, try again after a wait period delay_restart(failures) #---------------------------------------------------------------------- linkTable = linkTableMod # reinstate temporarily disabled links rows = npy.where(linkTable[:,cfg.LTB_LINKTYPE] > 1000) linkTable[rows,cfg.LTB_LINKTYPE] = (linkTable[rows,cfg.LTB_LINKTYPE] - 1000) # Drop links that are too long DISABLE_LEAST_COST_NO_VAL = True linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, cfg.MINEUCDIST, cfg.MAXCOSTDIST, cfg.MINCOSTDIST, DISABLE_LEAST_COST_NO_VAL) # Write link table file outlinkTableFile = lu.get_this_step_link_table(step=3) gprint('Updating ' + outlinkTableFile) lu.write_link_table(linkTable, outlinkTableFile) linkTableLogFile = path.join(cfg.LOGDIR, "linkTable_s3.csv") lu.write_link_table(linkTable, linkTableLogFile) start_time = time.clock() gprint('Creating shapefiles with linework for links...') try: lu.write_link_maps(outlinkTableFile, step=3) except: lu.write_link_maps(outlinkTableFile, step=3) start_time = lu.elapsed_time(start_time) gprint('\nIndividual cost-weighted distance layers written ' 'to "cwd" directory. \n') gprint(outlinkTableFile + '\n updated with cost-weighted distances between core areas.') #Clean up temporary files for restart code tempFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") lu.delete_file(tempFile) tempFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.delete_file(tempFile) # Check if climate tool is calling linkage mapper if cfg.TOOL == cfg.TOOL_CC: coreList = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) for core in coreList: cwdRaster = lu.get_cwd_path(int(core)) back_rast = cwdRaster.replace("cwd_", "back_") lu.delete_data(back_rast) # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME) return
def doRadiusLoop(): linkTable = linkTableTemp.copy() startTime = time.clock() randomerror() linkLoop = 0 pctDone = 0 gprint('\nMapping barriers at a radius of ' + str(radius) + ' ' + str(mapUnits)) if cfg.SUM_BARRIERS: gprint('using SUM method') else: gprint('using MAXIMUM method') if numCorridorLinks > 1: gprint('0 percent done') lastMosaicRaster = None lastMosaicRasterPct = None for x in range(0, numLinks): pctDone = lu.report_pct_done(linkLoop, numCorridorLinks, pctDone) linkId = str(int(linkTable[x, cfg.LTB_LINKID])) if ((linkTable[x, cfg.LTB_LINKTYPE] > 0) and (linkTable[x, cfg.LTB_LINKTYPE] < 1000)): linkLoop = linkLoop + 1 # source and target cores corex = int(coreList[x, 0]) corey = int(coreList[x, 1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) # Mask out areas above CWD threshold cwdTemp1 = None cwdTemp2 = None if cfg.BARRIER_CWD_THRESH is not None: if x == 1: lu.dashline(1) gprint(' Using CWD threshold of ' + str(cfg.BARRIER_CWD_THRESH) + ' map units.') arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.env.snapRaster = cfg.RESRAST cwdTemp1 = path.join(cfg.SCRATCHDIR, "tmp" + str(corex)) outCon = arcpy.sa.Con( cwdRaster1 < float(cfg.BARRIER_CWD_THRESH), cwdRaster1) outCon.save(cwdTemp1) cwdRaster1 = cwdTemp1 cwdTemp2 = path.join(cfg.SCRATCHDIR, "tmp" + str(corey)) outCon = arcpy.sa.Con( cwdRaster2 < float(cfg.BARRIER_CWD_THRESH), cwdRaster2) outCon.save(cwdTemp2) cwdRaster2 = cwdTemp2 focalRaster1 = lu.get_focal_path(corex, radius) focalRaster2 = lu.get_focal_path(corey, radius) link = lu.get_links_from_core_pairs( linkTable, corex, corey) lcDist = float(linkTable[link, cfg.LTB_CWDIST]) # Detect barriers at radius using neighborhood stats # Create the Neighborhood Object innerRadius = radius - 1 outerRadius = radius dia = 2 * radius InNeighborhood = ("ANNULUS " + str(innerRadius) + " " + str(outerRadius) + " MAP") @retry(10) def execFocal(): randomerror() # Execute FocalStatistics if not path.exists(focalRaster1): arcpy.env.extent = cwdRaster1 outFocalStats = arcpy.sa.FocalStatistics( cwdRaster1, InNeighborhood, "MINIMUM", "DATA") if setCoresToNull: outFocalStats2 = arcpy.sa.Con( outFocalStats > 0, outFocalStats ) # Set areas overlapping cores to NoData xxx outFocalStats2.save(focalRaster1) #xxx else: outFocalStats.save(focalRaster1) #xxx arcpy.env.extent = cfg.RESRAST if not path.exists(focalRaster2): arcpy.env.extent = cwdRaster2 outFocalStats = arcpy.sa.FocalStatistics( cwdRaster2, InNeighborhood, "MINIMUM", "DATA") if setCoresToNull: outFocalStats2 = arcpy.sa.Con( outFocalStats > 0, outFocalStats ) # Set areas overlapping cores to NoData xxx outFocalStats2.save(focalRaster2) #xxx else: outFocalStats.save(focalRaster2) #xxx arcpy.env.extent = cfg.RESRAST execFocal() lu.delete_data(cwdTemp1) lu.delete_data(cwdTemp2) barrierRaster = path.join( cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey) + '.tif') if cfg.SUM_BARRIERS: # Need to set nulls to 0, also # create trim rasters as we go outRas = ((lcDist - Raster(focalRaster1) - Raster(focalRaster2) - dia) / dia) outCon = arcpy.sa.Con(IsNull(outRas), 0, outRas) outCon2 = arcpy.sa.Con(outCon < 0, 0, outCon) outCon2.save(barrierRaster) # Execute FocalStatistics to fill out search radii InNeighborhood = "CIRCLE " + str( outerRadius) + " MAP" fillRaster = path.join( cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey) + "_fill.tif") outFocalStats = arcpy.sa.FocalStatistics( barrierRaster, InNeighborhood, "MAXIMUM", "DATA") outFocalStats.save(fillRaster) if cfg.WRITE_TRIM_RASTERS: trmRaster = path.join( cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey) + "_trim.tif") rasterList = [fillRaster, resistFillRaster] outCellStatistics = arcpy.sa.CellStatistics( rasterList, "MINIMUM") outCellStatistics.save(trmRaster) else: #Calculate potential benefit per map unit restored @retry(10) def calcBen(): randomerror() outRas = ((lcDist - Raster(focalRaster1) - Raster(focalRaster2) - dia) / dia) outRas.save(barrierRaster) calcBen() if cfg.WRITE_PCT_RASTERS: #Calculate PERCENT potential benefit per unit restored barrierRasterPct = path.join( cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey) + '_pct.tif') @retry(10) def calcBenPct(): randomerror() outras = (100 * (Raster(barrierRaster) / lcDist)) outras.save(barrierRasterPct) calcBenPct() # Mosaic barrier results across core area pairs mosaicDir = path.join( cfg.SCRATCHDIR, 'mos' + str(radId) + '_' + str(x + 1)) lu.create_dir(mosaicDir) mosFN = 'mos_temp' tempMosaicRaster = path.join(mosaicDir, mosFN) tempMosaicRasterTrim = path.join( mosaicDir, 'mos_temp_trm') arcpy.env.workspace = mosaicDir if linkLoop == 1: #If this is the first grid then copy rather than mosaic arcpy.CopyRaster_management( barrierRaster, tempMosaicRaster) if cfg.SUM_BARRIERS and cfg.WRITE_TRIM_RASTERS: arcpy.CopyRaster_management( trmRaster, tempMosaicRasterTrim) else: if cfg.SUM_BARRIERS: outCon = arcpy.sa.Con( Raster(barrierRaster) < 0, lastMosaicRaster, Raster(barrierRaster) + Raster(lastMosaicRaster)) outCon.save(tempMosaicRaster) if cfg.WRITE_TRIM_RASTERS: outCon = arcpy.sa.Con( Raster(trmRaster) < 0, lastMosaicRasterTrim, Raster(trmRaster) + Raster(lastMosaicRasterTrim)) outCon.save(tempMosaicRasterTrim) else: rasterString = ('"' + barrierRaster + ";" + lastMosaicRaster + '"') @retry(10) def mosaicToNew(): randomerror() arcpy.MosaicToNewRaster_management( rasterString, mosaicDir, mosFN, "", "32_BIT_FLOAT", arcpy.env.cellSize, "1", "MAXIMUM", "MATCH") mosaicToNew() # gprint(str(corex)+'0'+str(corey)) if linkLoop > 1: #Clean up from previous loop lu.delete_data(lastMosaicRaster) lastMosaicDir = path.dirname(lastMosaicRaster) lu.clean_out_workspace(lastMosaicDir) lu.delete_dir(lastMosaicDir) lastMosaicRaster = tempMosaicRaster if cfg.WRITE_TRIM_RASTERS: lastMosaicRasterTrim = tempMosaicRasterTrim if cfg.WRITE_PCT_RASTERS: mosPctFN = 'mos_temp_pct' mosaicDirPct = path.join( cfg.SCRATCHDIR, 'mosP' + str(radId) + '_' + str(x + 1)) lu.create_dir(mosaicDirPct) tempMosaicRasterPct = path.join( mosaicDirPct, mosPctFN) if linkLoop == 1: # If this is the first grid then copy # rather than mosaic if cfg.SUM_BARRIERS: outCon = arcpy.sa.Con( Raster(barrierRasterPct) < 0, 0, arcpy.sa.Con(IsNull(barrierRasterPct), 0, barrierRasterPct)) outCon.save(tempMosaicRasterPct) else: arcpy.CopyRaster_management( barrierRasterPct, tempMosaicRasterPct) else: if cfg.SUM_BARRIERS: @retry(10) def sumBarriers(): randomerror() outCon = arcpy.sa.Con( Raster(barrierRasterPct) < 0, lastMosaicRasterPct, Raster(barrierRasterPct) + Raster(lastMosaicRasterPct)) outCon.save(tempMosaicRasterPct) sumBarriers() else: rasterString = ('"' + barrierRasterPct + ";" + lastMosaicRasterPct + '"') @retry(10) def maxBarriers(): randomerror() arcpy.MosaicToNewRaster_management( rasterString, mosaicDirPct, mosPctFN, "", "32_BIT_FLOAT", arcpy.env.cellSize, "1", "MAXIMUM", "MATCH") maxBarriers() if linkLoop > 1: #Clean up from previous loop lu.delete_data(lastMosaicRasterPct) lastMosaicDirPct = path.dirname( lastMosaicRasterPct) lu.clean_out_workspace(lastMosaicDirPct) lu.delete_dir(lastMosaicDirPct) # lu.delete_data(lastMosaicRasterPct) lastMosaicRasterPct = tempMosaicRasterPct if not cfg.SAVEBARRIERRASTERS: lu.delete_data(barrierRaster) if cfg.WRITE_PCT_RASTERS: lu.delete_data(barrierRasterPct) if cfg.WRITE_TRIM_RASTERS: lu.delete_data(trmRaster) # Temporarily disable links in linktable - # don't want to mosaic them twice for y in range(x + 1, numLinks): corex1 = int(coreList[y, 0]) corey1 = int(coreList[y, 1]) if corex1 == corex and corey1 == corey: linkTable[y, cfg.LTB_LINKTYPE] = ( linkTable[y, cfg.LTB_LINKTYPE] + 1000) elif corex1 == corey and corey1 == corex: linkTable[y, cfg.LTB_LINKTYPE] = ( linkTable[y, cfg.LTB_LINKTYPE] + 1000) if numCorridorLinks > 1 and pctDone < 100: gprint('100 percent done') gprint('Summarizing barrier data for search radius.') #rows that were temporarily disabled rows = npy.where(linkTable[:, cfg.LTB_LINKTYPE] > 1000) linkTable[rows, cfg.LTB_LINKTYPE] = ( linkTable[rows, cfg.LTB_LINKTYPE] - 1000) # ----------------------------------------------------------------- # Set negative values to null or zero and write geodatabase. mosaicFN = (PREFIX + "_BarrierCenters" + sumSuffix + "_Rad" + str(radius)) mosaicRaster = path.join(cfg.BARRIERGDB, mosaicFN) arcpy.env.extent = cfg.RESRAST # if setCoresToNull: # outCon = arcpy.sa.Con(Raster(tempMosaicRaster) < 0, 0, # tempMosaicRaster) #xxx # outCon.save(mosaicRaster) #xxx # else: outSetNull = arcpy.sa.SetNull(tempMosaicRaster, tempMosaicRaster, "VALUE < 0") #xxx orig outSetNull.save(mosaicRaster) lu.delete_data(tempMosaicRaster) if cfg.SUM_BARRIERS and cfg.WRITE_TRIM_RASTERS: mosaicFN = (PREFIX + "_BarrierCircles_RBMin" + sumSuffix + "_Rad" + str(radius)) mosaicRasterTrim = path.join(cfg.BARRIERGDB, mosaicFN) arcpy.CopyRaster_management(tempMosaicRasterTrim, mosaicRasterTrim) lu.delete_data(tempMosaicRaster) if cfg.WRITE_PCT_RASTERS: # Do same for percent raster mosaicPctFN = (PREFIX + "_BarrierCenters_Pct" + sumSuffix + "_Rad" + str(radius)) arcpy.env.extent = cfg.RESRAST outSetNull = arcpy.sa.SetNull(tempMosaicRasterPct, tempMosaicRasterPct, "VALUE < 0") mosaicRasterPct = path.join(cfg.BARRIERGDB, mosaicPctFN) outSetNull.save(mosaicRasterPct) lu.delete_data(tempMosaicRasterPct) # 'Grow out' maximum restoration gain to # neighborhood size for display InNeighborhood = "CIRCLE " + str(outerRadius) + " MAP" # Execute FocalStatistics fillRasterFN = "barriers_fill" + str(outerRadius) + tif fillRaster = path.join(cfg.BARRIERBASEDIR, fillRasterFN) outFocalStats = arcpy.sa.FocalStatistics( mosaicRaster, InNeighborhood, "MAXIMUM", "DATA") outFocalStats.save(fillRaster) if cfg.WRITE_PCT_RASTERS: # Do same for percent raster fillRasterPctFN = "barriers_fill_pct" + str( outerRadius) + tif fillRasterPct = path.join(cfg.BARRIERBASEDIR, fillRasterPctFN) outFocalStats = arcpy.sa.FocalStatistics( mosaicRasterPct, InNeighborhood, "MAXIMUM", "DATA") outFocalStats.save(fillRasterPct) #Place copies of filled rasters in output geodatabase arcpy.env.workspace = cfg.BARRIERGDB fillRasterFN = (PREFIX + "_BarrrierCircles" + sumSuffix + "_Rad" + str(outerRadius)) arcpy.CopyRaster_management(fillRaster, fillRasterFN) if cfg.WRITE_PCT_RASTERS: fillRasterPctFN = (PREFIX + "_BarrrierCircles_Pct" + sumSuffix + "_Rad" + str(outerRadius)) arcpy.CopyRaster_management(fillRasterPct, fillRasterPctFN) if not cfg.SUM_BARRIERS and cfg.WRITE_TRIM_RASTERS: # Create pared-down version of filled raster- remove pixels # that don't need restoring by allowing a pixel to only # contribute its resistance value to restoration gain outRasterFN = "barriers_trm" + str(outerRadius) + tif outRaster = path.join(cfg.BARRIERBASEDIR, outRasterFN) rasterList = [fillRaster, resistFillRaster] outCellStatistics = arcpy.sa.CellStatistics( rasterList, "MINIMUM") outCellStatistics.save(outRaster) #SECOND ROUND TO CLIP BY DATA VALUES IN BARRIER RASTER outRaster2FN = ("barriers_trm" + sumSuffix + str(outerRadius) + "_2" + tif) outRaster2 = path.join(cfg.BARRIERBASEDIR, outRaster2FN) output = arcpy.sa.Con(IsNull(fillRaster), fillRaster, outRaster) output.save(outRaster2) outRasterFN = (PREFIX + "_BarrierCircles_RBMin" + sumSuffix + "_Rad" + str(outerRadius)) outRasterPath = path.join(cfg.BARRIERGDB, outRasterFN) arcpy.CopyRaster_management(outRaster2, outRasterFN) randomerror() startTime = lu.elapsed_time(startTime)
def STEP8_calc_pinchpoints(): """ Maps pinch points in Linkage Mapper corridors using Circuitscape given CWD calculations from s3_calcCwds.py. """ try: lu.dashline(0) gprint('Running script ' + _SCRIPT_NAME) restartFlag = False if cfg.CWDCUTOFF < 0: cfg.CWDCUTOFF = cfg.CWDCUTOFF * -1 restartFlag = True # Restart code in progress CSPATH = lu.get_cs_path() outputGDB = path.join(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) arcpy.OverWriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.pyramid = "NONE" arcpy.env.rasterstatistics = "NONE" # set the analysis extent and cell size to that of the resistance # surface arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.snapraster = cfg.RESRAST resRaster = cfg.RESRAST arcpy.env.extent = "MINOF" minObject = arcpy.GetRasterProperties_management(resRaster, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) if rasterMin <= 0: msg = ('Error: resistance raster cannot have 0 or negative values.') lu.raise_error(msg) if cfg.DO_ADJACENTPAIRS: prevLcpShapefile = lu.get_lcp_shapefile(None, thisStep = 8) if not arcpy.Exists(prevLcpShapefile): msg = ('Cannot find an LCP shapefile from step 5. Please ' 'rerun that step and any previous ones if necessary.') lu.raise_error(msg) # Remove lcp shapefile lcpShapefile = path.join(cfg.DATAPASSDIR, "lcpLines_s8.shp") lu.delete_data(lcpShapefile) inLinkTableFile = lu.get_prev_step_link_table(step=8) linkTable = lu.load_link_table(inLinkTableFile) numLinks = linkTable.shape[0] numCorridorLinks = lu.report_links(linkTable) if numCorridorLinks == 0: lu.dashline(1) msg =('\nThere are no linkages. Bailing.') lu.raise_error(msg) if linkTable.shape[1] < 16: # If linktable has no entries from prior # centrality or pinchpint analyses extraCols = npy.zeros((numLinks, 6), dtype="float64") linkTable = linkTable[:,0:10] linkTable = npy.append(linkTable, extraCols, axis=1) linkTable[:, cfg.LTB_LCPLEN] = -1 linkTable[:, cfg.LTB_CWDEUCR] = -1 linkTable[:, cfg.LTB_CWDPATHR] = -1 linkTable[:, cfg.LTB_EFFRESIST] = -1 linkTable[:, cfg.LTB_CWDTORR] = -1 linkTable[:, cfg.LTB_CURRENT] = -1 del extraCols # set up directories for circuit and circuit mosaic grids # Create output geodatabase if not arcpy.Exists(cfg.PINCHGDB): arcpy.CreateFileGDB_management(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) mosaicRaster = path.join(cfg.CIRCUITBASEDIR, "current_mos" + tif) coresToProcess = npy.unique( linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) maxCoreNum = max(coresToProcess) del coresToProcess lu.dashline(0) coreList = linkTable[:,cfg.LTB_CORE1:cfg.LTB_CORE2+1] coreList = npy.sort(coreList) #gprint('There are ' + str(len(npy.unique(coreList))) ' core areas.') INCIRCUITDIR = cfg.CIRCUITBASEDIR OUTCIRCUITDIR = path.join(cfg.CIRCUITBASEDIR, cfg.CIRCUITOUTPUTDIR_NM) CONFIGDIR = path.join(INCIRCUITDIR, cfg.CIRCUITCONFIGDIR_NM) # Cutoff value text to append to filenames cutoffText = str(cfg.CWDCUTOFF) if cutoffText[-6:] == '000000': cutoffText = cutoffText[0:-6]+'m' elif cutoffText[-3:] == '000': cutoffText = cutoffText[0:-3]+'k' if cfg.SQUARERESISTANCES: # Square resistance values squaredRaster = path.join(cfg.SCRATCHDIR,'res_sqr') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR outRas = Raster(resRaster) * Raster(resRaster) outRas.save(squaredRaster) resRaster = squaredRaster if cfg.DO_ADJACENTPAIRS: linkLoop = 0 lu.dashline(1) gprint('Mapping pinch points in individual corridors \n' 'using Circuitscape.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(2) for x in range(0,numLinks): linkId = str(int(linkTable[x,cfg.LTB_LINKID])) if not (linkTable[x,cfg.LTB_LINKTYPE] > 0): continue linkLoop = linkLoop + 1 linkDir = path.join(cfg.SCRATCHDIR, 'link' + linkId) if restartFlag == True and path.exists(linkDir): gprint('continuing') continue restartFlag = False lu.create_dir(linkDir) start_time1 = time.clock() # source and target cores corex=int(coreList[x,0]) corey=int(coreList[x,1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) lccNormRaster = path.join(linkDir, 'lcc_norm') arcpy.env.extent = "MINOF" link = lu.get_links_from_core_pairs(linkTable, corex, corey) lcDist = float(linkTable[link,cfg.LTB_CWDIST]) # Normalized lcc rasters are created by adding cwd rasters # and subtracting the least cost distance between them. outRas = Raster(cwdRaster1) + Raster(cwdRaster2) - lcDist outRas.save(lccNormRaster) #create raster mask resMaskRaster = path.join(linkDir, 'res_mask'+tif) #create raster mask outCon = arcpy.sa.Con(Raster(lccNormRaster) <= cfg.CWDCUTOFF, 1) outCon.save(resMaskRaster) # Convert to poly. Use as mask to clip resistance raster. resMaskPoly = path.join(linkDir, 'res_mask_poly.shp') arcpy.RasterToPolygon_conversion(resMaskRaster, resMaskPoly, "NO_SIMPLIFY") arcpy.env.extent = resMaskPoly # Includes 0 values in some cases with CP LI model if tif # so using ESRI Grid format resClipRasterMasked = path.join(linkDir, 'res_clip_m') # Extract masked resistance raster. # Needs to be float to get export to npy to work. outRas = arcpy.sa.ExtractByMask(resRaster, resMaskPoly) + 0.0 outRas.save(resClipRasterMasked) resNpyFN = 'resistances_link_' + linkId + '.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy(resClipRasterMasked, resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) gwarn('Warning:') gwarn('Circuitscape can only solve 2-3 million nodes') gwarn('per gigabyte of available RAM. \nTotal physical RAM' ' on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~'+ str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(2) corePairRaster = path.join(linkDir, 'core_pairs'+tif) arcpy.env.extent = resClipRasterMasked # Next result needs to be floating pt for numpy export outCon = arcpy.sa.Con(Raster(cwdRaster1) == 0, corex, arcpy.sa.Con(Raster(cwdRaster2) == 0, corey + 0.0)) outCon.save(corePairRaster) coreNpyFN = 'cores_link_' + linkId + '.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy(corePairRaster, coreNpyFile) arcpy.env.extent = "MINOF" # Set circuitscape options and call options = lu.setCircuitscapeOptions() if cfg.WRITE_VOLT_MAPS == True: options['write_volt_maps']=True options['habitat_file'] = resNpyFile # if int(linkId) > 2: # options['habitat_file'] = 'c:\\test.dummy' options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero']=True outputFN = 'Circuitscape_link' + linkId + '.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) if numElements > 250000: options['print_timings']=True configFN = 'pinchpoint_config' + linkId + '.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('Processing link ID #' + str(linkId) + '. Resistance map' ' has ' + str(int(numResistanceNodes)) + ' nodes.') memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): print_failure(numResistanceNodes, memFlag, 10) numElements, numNodes = export_ras_to_npy( resClipRasterMasked,resNpyFile) memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): msg = ('\nCircuitscape failed. See error information above.') arcpy.AddError(msg) lu.write_log(msg) exit(1) # Either set core areas to nodata in current map or # divide each by its radius currentRaster = path.join(linkDir, "current" + tif) import_npy_to_ras(currentMap,corePairRaster,currentRaster) if cfg.WRITE_VOLT_MAPS == True: voltFN = ('Circuitscape_link' + linkId + '_voltmap_' + str(corex) + '_'+str(corey) + '.npy') voltMap = path.join(OUTCIRCUITDIR, voltFN) voltRaster = path.join(outputGDB, cfg.PREFIX + "_voltMap_"+ str(corex) + '_'+str(corey)) import_npy_to_ras(voltMap,corePairRaster,voltRaster) gprint('Building output statistics and pyramids ' 'for voltage raster\n') lu.build_stats(voltRaster) arcpy.env.extent = currentRaster if SETCORESTONULL: # Set core areas to NoData in current map for color ramping currentRaster2 = currentRaster + '2' + tif outCon = arcpy.sa.Con(arcpy.sa.IsNull(Raster (corePairRaster)), Raster(currentRaster)) outCon.save(currentRaster2) currentRaster = currentRaster2 arcpy.env.extent = "MAXOF" if linkLoop == 1: lu.delete_data(mosaicRaster) @retry(10) def copyRas2(): arcpy.CopyRaster_management(currentRaster, mosaicRaster) copyRas2() else: @retry(10) def mosaicRas(): arcpy.Mosaic_management(currentRaster, mosaicRaster, "MAXIMUM", "MATCH") mosaicRas() resistancesFN = ('Circuitscape_link' + linkId + '_resistances_3columns.out') resistancesFile = path.join(OUTCIRCUITDIR,resistancesFN) resistances = npy.loadtxt(resistancesFile, dtype = 'Float64', comments='#') resistance = float(str(arcpy.env.cellSize)) * resistances[2] linkTable[link,cfg.LTB_EFFRESIST] = resistance # Ratio if not cfg.SQUARERESISTANCES: linkTable[link,cfg.LTB_CWDTORR] = (linkTable[link, cfg.LTB_CWDIST] / linkTable[link,cfg.LTB_EFFRESIST]) # Clean up if cfg.SAVE_TEMP_CIRCUIT_FILES == False: lu.delete_file(coreNpyFile) coreNpyBase, extension = path.splitext(coreNpyFile) lu.delete_data(coreNpyBase + '.hdr') lu.delete_file(resNpyFile) resNpyBase, extension = path.splitext(resNpyFile) lu.delete_data(resNpyBase + '.hdr') lu.delete_file(currentMap) curMapBase, extension = path.splitext(currentMap) lu.delete_data(curMapBase + '.hdr') lu.delete_data(currentRaster) lu.clean_out_workspace(linkDir) lu.delete_dir(linkDir) gprint('Finished with link ID #' + str(linkId) + '. ' + str(linkLoop) + ' out of ' + str(numCorridorLinks) + ' links have been processed.') start_time1 = lu.elapsed_time(start_time1) outputRaster = path.join(outputGDB, cfg.PREFIX + "_current_adjacentPairs_" + cutoffText) lu.delete_data(outputRaster) @retry(10) def copyRas(): arcpy.CopyRaster_management(mosaicRaster, outputRaster) copyRas() gprint('Building output statistics and pyramids ' 'for corridor pinch point raster\n') lu.build_stats(outputRaster) finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=5, thisStep=8) linkTableFile = path.join(cfg.DATAPASSDIR, "linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFile, inLinkTableFile) linkTableFinalFile = path.join(cfg.OUTPUTDIR, cfg.PREFIX + "_linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFinalFile, inLinkTableFile) gprint('Copy of linkTable written to '+ linkTableFinalFile) #fixme: update sticks? gprint('Creating shapefiles with linework for links.') lu.write_link_maps(linkTableFinalFile, step=8) # Copy final link maps to gdb. lu.copy_final_link_maps(step=8) lu.delete_data(mosaicRaster) if not cfg.DO_ALLPAIRS: # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) return lu.dashline(1) gprint('Mapping global pinch points among all\n' 'core area pairs using Circuitscape.') if cfg.ALL_PAIR_SCENARIO=='pairwise': gprint('Circuitscape will be run in PAIRWISE mode.') else: gprint('Circuitscape will be run in ALL-TO-ONE mode.') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST S8CORE_RAS = "s8core_ras" s8CoreRasPath = path.join(cfg.SCRATCHDIR,S8CORE_RAS) arcpy.FeatureToRaster_conversion(cfg.COREFC, cfg.COREFN, s8CoreRasPath, arcpy.env.cellSize) binaryCoreRaster = path.join(cfg.SCRATCHDIR,"core_ras_bin") # The following commands cause file lock problems on save. using gp # instead. # outCon = arcpy.sa.Con(S8CORE_RAS, 1, "#", "VALUE > 0") # outCon.save(binaryCoreRaster) # gp.Con_sa(s8CoreRasPath, 1, binaryCoreRaster, "#", "VALUE > 0") outCon = arcpy.sa.Con(Raster(s8CoreRasPath) > 0, 1) outCon.save(binaryCoreRaster) s5corridorRas = path.join(cfg.OUTPUTGDB,cfg.PREFIX + "_corridors") if not arcpy.Exists(s5corridorRas): s5corridorRas = path.join(cfg.OUTPUTGDB,cfg.PREFIX + "_lcc_mosaic_int") outCon = arcpy.sa.Con(Raster(s5corridorRas) <= cfg.CWDCUTOFF, Raster( resRaster), arcpy.sa.Con(Raster( binaryCoreRaster) > 0, Raster(resRaster))) resRasClipPath = path.join(cfg.SCRATCHDIR,'res_ras_clip') outCon.save(resRasClipPath) arcpy.env.cellSize = resRasClipPath arcpy.env.extent = resRasClipPath s8CoreRasClipped = s8CoreRasPath + '_c' # Produce core raster with same extent as clipped resistance raster # added to ensure correct data type- nodata values were positive for # cores otherwise outCon = arcpy.sa.Con(arcpy.sa.IsNull(Raster(s8CoreRasPath)), -9999, Raster(s8CoreRasPath)) outCon.save(s8CoreRasClipped) resNpyFN = 'resistances.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy(resRasClipPath,resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) gwarn('Warning:') gwarn('Circuitscape can only solve 2-3 million nodes') gwarn('per gigabyte of available RAM. \nTotal physical RAM ' 'on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~'+ str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(0) coreNpyFN = 'cores.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy(s8CoreRasClipped,coreNpyFile) arcpy.env.extent = "MINOF" options = lu.setCircuitscapeOptions() options['scenario']=cfg.ALL_PAIR_SCENARIO options['habitat_file'] = resNpyFile options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero']=True outputFN = 'Circuitscape.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) options['print_timings']=True configFN = 'pinchpoint_allpair_config.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('\nResistance map has ' + str(int(numResistanceNodes)) + ' nodes.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(0) call_circuitscape(CSPATH, outConfigFile) # test = subprocess.call([CSPATH, outConfigFile], # creationflags = subprocess.CREATE_NEW_CONSOLE) if options['scenario']=='pairwise': rasterSuffix = "_current_allPairs_" + cutoffText else: rasterSuffix = "_current_allToOne_" + cutoffText currentFN = 'Circuitscape_cum_curmap.npy' currentMap = path.join(OUTCIRCUITDIR, currentFN) outputRaster = path.join(outputGDB, cfg.PREFIX + rasterSuffix) currentRaster = path.join(cfg.SCRATCHDIR, "current") try: import_npy_to_ras(currentMap,resRasClipPath,outputRaster) except: lu.dashline(1) msg = ('ERROR: Circuitscape failed. \n' 'Note: Circuitscape can only solve 2-3 million nodes' '\nper gigabyte of available RAM. The resistance ' '\nraster for the last corridor had ' + str(numResistanceNodes) + ' nodes.\n\nResistance ' 'raster values that vary by >6 orders of \nmagnitude' ' can also cause failures, as can a mismatch in ' '\ncore area and resistance raster extents.') arcpy.AddError(msg) lu.write_log(msg) exit(1) #set core areas to nodata if SETCORESTONULL: # Set core areas to NoData in current map for color ramping outputRasterND = outputRaster + '_noDataCores' outCon = arcpy.sa.SetNull(Raster(s8CoreRasClipped) > 0, Raster(outputRaster)) outCon.save(outputRasterND) gprint('\nBuilding output statistics and pyramids ' 'for centrality raster.') lu.build_stats(outputRaster) lu.build_stats(outputRasterND) # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) # Return GEOPROCESSING specific errors except arcpy.ExecuteError: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME)
def doRadiusLoop(): linkTable = linkTableTemp.copy() startTime = time.clock() randomerror() linkLoop = 0 pctDone = 0 gprint('\nMapping barriers at a radius of ' + str(radius) + ' ' + str(mapUnits)) if cfg.SUM_BARRIERS: gprint('using SUM method') else: gprint('using MAXIMUM method') if numCorridorLinks > 1: gprint('0 percent done') lastMosaicRaster = None lastMosaicRasterPct = None for x in range(0,numLinks): pctDone = lu.report_pct_done(linkLoop, numCorridorLinks, pctDone) linkId = str(int(linkTable[x,cfg.LTB_LINKID])) if ((linkTable[x,cfg.LTB_LINKTYPE] > 0) and (linkTable[x,cfg.LTB_LINKTYPE] < 1000)): linkLoop = linkLoop + 1 # source and target cores corex=int(coreList[x,0]) corey=int(coreList[x,1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) # Mask out areas above CWD threshold cwdTemp1 = None cwdTemp2 = None if cfg.BARRIER_CWD_THRESH is not None: if x == 1: lu.dashline(1) gprint(' Using CWD threshold of ' + str(cfg.BARRIER_CWD_THRESH) + ' map units.') arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.env.snapRaster = cfg.RESRAST cwdTemp1 = path.join(cfg.SCRATCHDIR, "tmp"+str(corex)) outCon = arcpy.sa.Con(cwdRaster1 < float(cfg.BARRIER_CWD_THRESH),cwdRaster1) outCon.save(cwdTemp1) cwdRaster1 = cwdTemp1 cwdTemp2 = path.join(cfg.SCRATCHDIR, "tmp"+str(corey)) outCon = arcpy.sa.Con(cwdRaster2 < float(cfg.BARRIER_CWD_THRESH),cwdRaster2) outCon.save(cwdTemp2) cwdRaster2 = cwdTemp2 focalRaster1 = lu.get_focal_path(corex,radius) focalRaster2 = lu.get_focal_path(corey,radius) link = lu.get_links_from_core_pairs(linkTable, corex, corey) lcDist = float(linkTable[link,cfg.LTB_CWDIST]) # Detect barriers at radius using neighborhood stats # Create the Neighborhood Object innerRadius = radius - 1 outerRadius = radius dia = 2 * radius InNeighborhood = ("ANNULUS " + str(innerRadius) + " " + str(outerRadius) + " MAP") @retry(10) def execFocal(): randomerror() # Execute FocalStatistics if not path.exists(focalRaster1): arcpy.env.extent = cwdRaster1 outFocalStats = arcpy.sa.FocalStatistics(cwdRaster1, InNeighborhood, "MINIMUM","DATA") if setCoresToNull: outFocalStats2 = arcpy.sa.Con(outFocalStats > 0, outFocalStats) # Set areas overlapping cores to NoData xxx outFocalStats2.save(focalRaster1) #xxx else: outFocalStats.save(focalRaster1) #xxx arcpy.env.extent = cfg.RESRAST if not path.exists(focalRaster2): arcpy.env.extent = cwdRaster2 outFocalStats = arcpy.sa.FocalStatistics(cwdRaster2, InNeighborhood, "MINIMUM","DATA") if setCoresToNull: outFocalStats2 = arcpy.sa.Con(outFocalStats > 0, outFocalStats) # Set areas overlapping cores to NoData xxx outFocalStats2.save(focalRaster2)#xxx else: outFocalStats.save(focalRaster2) #xxx arcpy.env.extent = cfg.RESRAST execFocal() lu.delete_data(cwdTemp1) lu.delete_data(cwdTemp2) barrierRaster = path.join(cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey)+'.tif') if cfg.SUM_BARRIERS: # Need to set nulls to 0, also # create trim rasters as we go outRas = ((lcDist - Raster(focalRaster1) - Raster(focalRaster2) - dia) / dia) outCon = arcpy.sa.Con(IsNull(outRas),0,outRas) outCon2 = arcpy.sa.Con(outCon<0,0,outCon) outCon2.save(barrierRaster) # Execute FocalStatistics to fill out search radii InNeighborhood = "CIRCLE " + str(outerRadius) + " MAP" fillRaster = path.join(cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey) +"_fill.tif") outFocalStats = arcpy.sa.FocalStatistics(barrierRaster, InNeighborhood, "MAXIMUM","DATA") outFocalStats.save(fillRaster) if cfg.WRITE_TRIM_RASTERS: trmRaster = path.join(cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey) +"_trim.tif") rasterList = [fillRaster, resistFillRaster] outCellStatistics = arcpy.sa.CellStatistics( rasterList, "MINIMUM") outCellStatistics.save(trmRaster) else: #Calculate potential benefit per map unit restored @retry(10) def calcBen(): randomerror() outRas = ((lcDist - Raster(focalRaster1) - Raster(focalRaster2) - dia) / dia) outRas.save(barrierRaster) calcBen() if cfg.WRITE_PCT_RASTERS: #Calculate PERCENT potential benefit per unit restored barrierRasterPct = path.join(cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey)+'_pct.tif') @retry(10) def calcBenPct(): randomerror() outras = (100 * (Raster(barrierRaster) / lcDist)) outras.save(barrierRasterPct) calcBenPct() # Mosaic barrier results across core area pairs mosaicDir = path.join(cfg.SCRATCHDIR,'mos'+str(radId)+'_'+str(x+1)) lu.create_dir(mosaicDir) mosFN = 'mos_temp' tempMosaicRaster = path.join(mosaicDir,mosFN) tempMosaicRasterTrim = path.join(mosaicDir,'mos_temp_trm') arcpy.env.workspace = mosaicDir if linkLoop == 1: #If this is the first grid then copy rather than mosaic arcpy.CopyRaster_management(barrierRaster, tempMosaicRaster) if cfg.SUM_BARRIERS and cfg.WRITE_TRIM_RASTERS: arcpy.CopyRaster_management(trmRaster, tempMosaicRasterTrim) else: if cfg.SUM_BARRIERS: outCon = arcpy.sa.Con(Raster (barrierRaster) < 0, lastMosaicRaster, Raster(barrierRaster) + Raster(lastMosaicRaster)) outCon.save(tempMosaicRaster) if cfg.WRITE_TRIM_RASTERS: outCon = arcpy.sa.Con(Raster (trmRaster) < 0, lastMosaicRasterTrim, Raster(trmRaster) + Raster( lastMosaicRasterTrim)) outCon.save(tempMosaicRasterTrim) else: rasterString = ('"'+barrierRaster+";" + lastMosaicRaster+'"') @retry(10) def mosaicToNew(): randomerror() arcpy.MosaicToNewRaster_management( rasterString,mosaicDir,mosFN, "", "32_BIT_FLOAT", arcpy.env.cellSize, "1", "MAXIMUM", "MATCH") mosaicToNew() # gprint(str(corex)+'0'+str(corey)) if linkLoop>1: #Clean up from previous loop lu.delete_data(lastMosaicRaster) lastMosaicDir =path.dirname(lastMosaicRaster) lu.clean_out_workspace(lastMosaicDir) lu.delete_dir(lastMosaicDir) lastMosaicRaster = tempMosaicRaster if cfg.WRITE_TRIM_RASTERS: lastMosaicRasterTrim = tempMosaicRasterTrim if cfg.WRITE_PCT_RASTERS: mosPctFN = 'mos_temp_pct' mosaicDirPct = path.join(cfg.SCRATCHDIR,'mosP'+str(radId)+'_'+str(x+1)) lu.create_dir(mosaicDirPct) tempMosaicRasterPct = path.join(mosaicDirPct,mosPctFN) if linkLoop == 1: # If this is the first grid then copy # rather than mosaic if cfg.SUM_BARRIERS: outCon = arcpy.sa.Con(Raster(barrierRasterPct) < 0, 0, arcpy.sa.Con(IsNull( barrierRasterPct), 0, barrierRasterPct)) outCon.save(tempMosaicRasterPct) else: arcpy.CopyRaster_management(barrierRasterPct, tempMosaicRasterPct) else: if cfg.SUM_BARRIERS: @retry(10) def sumBarriers(): randomerror() outCon = arcpy.sa.Con(Raster(barrierRasterPct) < 0, lastMosaicRasterPct, Raster(barrierRasterPct) + Raster( lastMosaicRasterPct)) outCon.save(tempMosaicRasterPct) sumBarriers() else: rasterString = ('"' + barrierRasterPct + ";" + lastMosaicRasterPct + '"') @retry(10) def maxBarriers(): randomerror() arcpy.MosaicToNewRaster_management( rasterString,mosaicDirPct,mosPctFN, "", "32_BIT_FLOAT", arcpy.env.cellSize, "1", "MAXIMUM", "MATCH") maxBarriers() if linkLoop>1: #Clean up from previous loop lu.delete_data(lastMosaicRasterPct) lastMosaicDirPct =path.dirname(lastMosaicRasterPct) lu.clean_out_workspace(lastMosaicDirPct) lu.delete_dir(lastMosaicDirPct) # lu.delete_data(lastMosaicRasterPct) lastMosaicRasterPct = tempMosaicRasterPct if not cfg.SAVEBARRIERRASTERS: lu.delete_data(barrierRaster) if cfg.WRITE_PCT_RASTERS: lu.delete_data(barrierRasterPct) if cfg.WRITE_TRIM_RASTERS: lu.delete_data(trmRaster) # Temporarily disable links in linktable - # don't want to mosaic them twice for y in range (x+1,numLinks): corex1 = int(coreList[y,0]) corey1 = int(coreList[y,1]) if corex1 == corex and corey1 == corey: linkTable[y,cfg.LTB_LINKTYPE] = ( linkTable[y,cfg.LTB_LINKTYPE] + 1000) elif corex1==corey and corey1==corex: linkTable[y,cfg.LTB_LINKTYPE] = ( linkTable[y,cfg.LTB_LINKTYPE] + 1000) if numCorridorLinks > 1 and pctDone < 100: gprint('100 percent done') gprint('Summarizing barrier data for search radius.') #rows that were temporarily disabled rows = npy.where(linkTable[:,cfg.LTB_LINKTYPE]>1000) linkTable[rows,cfg.LTB_LINKTYPE] = ( linkTable[rows,cfg.LTB_LINKTYPE] - 1000) # ----------------------------------------------------------------- # Set negative values to null or zero and write geodatabase. mosaicFN = (PREFIX + "_BarrierCenters" + sumSuffix + "_Rad" + str(radius)) mosaicRaster = path.join(cfg.BARRIERGDB, mosaicFN) arcpy.env.extent = cfg.RESRAST # if setCoresToNull: # outCon = arcpy.sa.Con(Raster(tempMosaicRaster) < 0, 0, # tempMosaicRaster) #xxx # outCon.save(mosaicRaster) #xxx # else: outSetNull = arcpy.sa.SetNull(tempMosaicRaster, tempMosaicRaster, "VALUE < 0") #xxx orig outSetNull.save(mosaicRaster) lu.delete_data(tempMosaicRaster) if cfg.SUM_BARRIERS and cfg.WRITE_TRIM_RASTERS: mosaicFN = (PREFIX + "_BarrierCircles_RBMin" + sumSuffix + "_Rad" + str(radius)) mosaicRasterTrim = path.join(cfg.BARRIERGDB, mosaicFN) arcpy.CopyRaster_management(tempMosaicRasterTrim, mosaicRasterTrim) lu.delete_data(tempMosaicRaster) if cfg.WRITE_PCT_RASTERS: # Do same for percent raster mosaicPctFN = (PREFIX + "_BarrierCenters_Pct" + sumSuffix + "_Rad" + str(radius)) arcpy.env.extent = cfg.RESRAST outSetNull = arcpy.sa.SetNull(tempMosaicRasterPct, tempMosaicRasterPct, "VALUE < 0") mosaicRasterPct = path.join(cfg.BARRIERGDB, mosaicPctFN) outSetNull.save(mosaicRasterPct) lu.delete_data(tempMosaicRasterPct) # 'Grow out' maximum restoration gain to # neighborhood size for display InNeighborhood = "CIRCLE " + str(outerRadius) + " MAP" # Execute FocalStatistics fillRasterFN = "barriers_fill" + str(outerRadius) + tif fillRaster = path.join(cfg.BARRIERBASEDIR, fillRasterFN) outFocalStats = arcpy.sa.FocalStatistics(mosaicRaster, InNeighborhood, "MAXIMUM","DATA") outFocalStats.save(fillRaster) if cfg.WRITE_PCT_RASTERS: # Do same for percent raster fillRasterPctFN = "barriers_fill_pct" + str(outerRadius) + tif fillRasterPct = path.join(cfg.BARRIERBASEDIR, fillRasterPctFN) outFocalStats = arcpy.sa.FocalStatistics(mosaicRasterPct, InNeighborhood, "MAXIMUM","DATA") outFocalStats.save(fillRasterPct) #Place copies of filled rasters in output geodatabase arcpy.env.workspace = cfg.BARRIERGDB fillRasterFN = (PREFIX + "_BarrrierCircles" + sumSuffix + "_Rad" + str(outerRadius)) arcpy.CopyRaster_management(fillRaster, fillRasterFN) if cfg.WRITE_PCT_RASTERS: fillRasterPctFN = (PREFIX + "_BarrrierCircles_Pct" + sumSuffix + "_Rad" + str(outerRadius)) arcpy.CopyRaster_management(fillRasterPct, fillRasterPctFN) if not cfg.SUM_BARRIERS and cfg.WRITE_TRIM_RASTERS: # Create pared-down version of filled raster- remove pixels # that don't need restoring by allowing a pixel to only # contribute its resistance value to restoration gain outRasterFN = "barriers_trm" + str(outerRadius) + tif outRaster = path.join(cfg.BARRIERBASEDIR,outRasterFN) rasterList = [fillRaster, resistFillRaster] outCellStatistics = arcpy.sa.CellStatistics(rasterList, "MINIMUM") outCellStatistics.save(outRaster) #SECOND ROUND TO CLIP BY DATA VALUES IN BARRIER RASTER outRaster2FN = ("barriers_trm" + sumSuffix + str(outerRadius) + "_2" + tif) outRaster2 = path.join(cfg.BARRIERBASEDIR,outRaster2FN) output = arcpy.sa.Con(IsNull(fillRaster),fillRaster,outRaster) output.save(outRaster2) outRasterFN = (PREFIX + "_BarrierCircles_RBMin" + sumSuffix + "_Rad" + str(outerRadius)) outRasterPath= path.join(cfg.BARRIERGDB, outRasterFN) arcpy.CopyRaster_management(outRaster2, outRasterFN) randomerror() startTime=lu.elapsed_time(startTime)
def gen_cwd_back(core_list, climate_lyr, resist_lyr, core_lyr): """"Generate CWD and back rasters using r.walk in GRASS""" slope_factor = "1" walk_coeff_flat = "1" walk_coeff_uphill = str(cc_env.climate_cost) walk_coeff_downhill = str(cc_env.climate_cost * -1) walk_coeff = (walk_coeff_flat + "," + walk_coeff_uphill + "," + walk_coeff_downhill + "," + walk_coeff_downhill) focal_core_rast = "focal_core_rast" gcwd = "gcwd" gback = "gback" gbackrc = "gbackrc" core_points = "corepoints" no_cores = str(len(core_list)) # Map from directional degree output from GRASS to Arc's 1 to 8 directions # format. See r.walk source code and ArcGIS's 'Understanding cost distance # analysis' help page. rc_rules = "180=5\n225=4\n270=3\n315=2\n360=1\n45=8\n90=7\n135=6" try: for position, core_no in enumerate(core_list): core_no_txt = str(core_no) lm_util.gprint("Generating CWD and back rasters for Core " + core_no_txt + " (" + str(position + 1) + "/" + no_cores + ")") # Pull out focal core for cwd analysis write_grass_cmd("r.reclass", input=core_lyr, output=focal_core_rast, overwrite=True, rules="-", stdin=core_no_txt + '=' + core_no_txt) # Converting raster core to point feature run_grass_cmd("r.to.vect", flags="z", input=focal_core_rast, output=core_points, type="point") # Running r.walk to create CWD and back raster run_grass_cmd("r.walk", elevation=climate_lyr, friction=resist_lyr, output=gcwd, outdir=gback, start_points=core_points, walk_coeff=walk_coeff, slope_factor=slope_factor) # Reclassify back raster directional degree output to ArcGIS format write_grass_cmd("r.reclass", input=gback, output=gbackrc, rules="-", stdin=rc_rules) # Get spatial reference for defining ARCINFO raster projections desc_data = arcpy.Describe(cc_env.prj_core_rast) spatial_ref = desc_data.spatialReference # Get cwd path (e.g. ..\datapass\cwd\cw\cwd_3) cwd_path = lm_util.get_cwd_path(core_no) def create_arcgrid(rtype, grass_grid): """Export GRASS raster to ASCII grid and then to ARCINFO grid """ ascii_grid = os.path.join(cc_env.out_dir, rtype + core_no_txt + ".asc") arc_grid = cwd_path.replace("cwd_", rtype) run_grass_cmd("r.out.gdal", input=grass_grid, output=ascii_grid, format="AAIGrid") arcpy.CopyRaster_management(ascii_grid, arc_grid) arcpy.DefineProjection_management(arc_grid, spatial_ref) os.remove(ascii_grid) create_arcgrid("cwd_", gcwd) # Export CWD raster create_arcgrid("back_", gbackrc) # Export reclassified back raster except Exception: raise
def calc_lccs(normalize): try: if normalize: mosaicBaseName = "_corridors" writeTruncRaster = cfg.WRITETRUNCRASTER outputGDB = cfg.OUTPUTGDB if cfg.CALCNONNORMLCCS: SAVENORMLCCS = False else: SAVENORMLCCS = cfg.SAVENORMLCCS else: mosaicBaseName = "_NON_NORMALIZED_corridors" SAVENORMLCCS = False outputGDB = cfg.EXTRAGDB writeTruncRaster = False lu.dashline(1) gprint('Running script ' + _SCRIPT_NAME) linkTableFile = lu.get_prev_step_link_table(step=5) if cfg.useArcpy: arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.overwriteOutput = True arcpy.env.compression = "NONE" else: gp.workspace = cfg.SCRATCHDIR gp.scratchWorkspace = cfg.ARCSCRATCHDIR gp.OverwriteOutput = True if cfg.MAXEUCDIST is not None: gprint('Max Euclidean distance between cores') gprint('for linkage mapping set to ' + str(cfg.MAXEUCDIST)) if cfg.MAXCOSTDIST is not None: gprint('Max cost-weighted distance between cores') gprint('for linkage mapping set to ' + str(cfg.MAXCOSTDIST)) # set the analysis extent and cell size to that of the resistance # surface if cfg.useArcpy: arcpy.env.Extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.env.snapRaster = cfg.RESRAST arcpy.env.mask = cfg.RESRAST else: gp.Extent = (gp.Describe(cfg.RESRAST)).Extent gp.cellSize = gp.Describe(cfg.RESRAST).MeanCellHeight gp.mask = cfg.RESRAST gp.snapraster = cfg.RESRAST linkTable = lu.load_link_table(linkTableFile) numLinks = linkTable.shape[0] numCorridorLinks = lu.report_links(linkTable) if numCorridorLinks == 0: lu.dashline(1) msg =('\nThere are no corridors to map. Bailing.') lu.raise_error(msg) if not cfg.STEP3 and not cfg.STEP4: # re-check for links that are too long or in case script run out of # sequence with more stringent settings gprint('Double-checking for corridors that are too long to map.') DISABLE_LEAST_COST_NO_VAL = True linkTable,numDroppedLinks = lu.drop_links( linkTable, cfg.MAXEUCDIST, cfg.MINEUCDIST, cfg.MAXCOSTDIST, cfg.MINCOSTDIST, DISABLE_LEAST_COST_NO_VAL) # Added to try to speed up: gp.pyramid = "NONE" gp.rasterstatistics = "NONE" # set up directories for normalized lcc and mosaic grids dirCount = 0 gprint("Creating output folder: " + cfg.LCCBASEDIR) lu.delete_dir(cfg.LCCBASEDIR) gp.CreateFolder_management(path.dirname(cfg.LCCBASEDIR), path.basename(cfg.LCCBASEDIR)) gp.CreateFolder_management(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM) clccdir = path.join(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM) # mosaicGDB = path.join(cfg.LCCBASEDIR, "mosaic.gdb") # gp.createfilegdb(cfg.LCCBASEDIR, "mosaic.gdb") #mosaicRaster = mosaicGDB + '\\' + "nlcc_mos" # Full path gprint("") if normalize: gprint('Normalized least-cost corridors will be written ' 'to ' + clccdir + '\n') PREFIX = cfg.PREFIX # Add CWD layers for core area pairs to produce NORMALIZED LCC layers numGridsWritten = 0 coreList = linkTable[:,cfg.LTB_CORE1:cfg.LTB_CORE2+1] coreList = npy.sort(coreList) x = 0 linkCount = 0 endIndex = numLinks while x < endIndex: if (linkTable[x, cfg.LTB_LINKTYPE] < 1): # If not a valid link x = x + 1 continue linkCount = linkCount + 1 start_time = time.clock() linkId = str(int(linkTable[x, cfg.LTB_LINKID])) # source and target cores corex=int(coreList[x,0]) corey=int(coreList[x,1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) if not gp.Exists(cwdRaster1): msg =('\nError: cannot find cwd raster:\n' + cwdRaster1) if not gp.Exists(cwdRaster2): msg =('\nError: cannot find cwd raster:\n' + cwdRaster2) lu.raise_error(msg) lccNormRaster = path.join(clccdir, str(corex) + "_" + str(corey))# + ".tif") if cfg.useArcpy: arcpy.env.Extent = "MINOF" else: gp.Extent = "MINOF" # FIXME: need to check for this?: # if exists already, don't re-create #if not gp.Exists(lccRaster): link = lu.get_links_from_core_pairs(linkTable, corex, corey) offset = 10000 # Normalized lcc rasters are created by adding cwd rasters and # subtracting the least cost distance between them. count = 0 if arcpyAvailable: cfg.useArcpy = True # Fixes Canran Liu's bug with lcDist if cfg.useArcpy: lcDist = (float(linkTable[link,cfg.LTB_CWDIST]) - offset) if normalize: statement = ('outras = Raster(cwdRaster1) + Raster(' 'cwdRaster2) - lcDist; outras.save(lccNormRaster)') else: statement = ('outras =Raster(cwdRaster1) + Raster(' 'cwdRaster2); outras.save(lccNormRaster)') else: if normalize: lcDist = str(linkTable[link,cfg.LTB_CWDIST] - offset) expression = (cwdRaster1 + " + " + cwdRaster2 + " - " + lcDist) else: expression = (cwdRaster1 + " + " + cwdRaster2) statement = ('gp.SingleOutputMapAlgebra_sa(expression, ' 'lccNormRaster)') count = 0 while True: try: exec statement randomerror() except: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec statement else: break cfg.useArcpy = False # End fix for Conran Liu's bug with lcDist if normalize and cfg.useArcpy: try: minObject = gp.GetRasterProperties(lccNormRaster, "MINIMUM") rasterMin = float(str(minObject.getoutput(0))) except: gp.AddWarning('\n------------------------------------------------') gp.AddWarning('WARNING: Raster minimum check failed in step 5. \n' 'This may mean the output rasters are corrupted. Please \n' 'be sure to check for valid rasters in '+ outputGDB) rasterMin = 0 tolerance = (float(gp.cellSize) * -10) + offset if rasterMin < tolerance: lu.dashline(1) msg = ('WARNING: Minimum value of a corridor #' + str(x+1) + ' is much less than zero ('+str(rasterMin)+').' '\nThis could mean that BOUNDING CIRCLE BUFFER DISTANCES ' 'were too small and a corridor passed outside of a ' 'bounding circle, or that a corridor passed outside of the ' 'resistance map. \n') gp.AddWarning(msg) if cfg.useArcpy: arcpy.env.Extent = cfg.RESRAST else: gp.Extent = (gp.Describe(cfg.RESRAST)).Extent mosaicDir = path.join(cfg.LCCBASEDIR,'mos'+str(x+1)) lu.create_dir(mosaicDir) mosFN = 'mos'#.tif' change and move mosaicRaster = path.join(mosaicDir,mosFN) if numGridsWritten == 0 and dirCount == 0: #If this is the first grid then copy rather than mosaic arcObj.CopyRaster_management(lccNormRaster, mosaicRaster) else: rasterString = '"'+lccNormRaster+";"+lastMosaicRaster+'"' statement = ('arcObj.MosaicToNewRaster_management(' 'rasterString,mosaicDir,mosFN, "", ' '"32_BIT_FLOAT", gp.cellSize, "1", "MINIMUM", ' '"MATCH")') # statement = ('arcpy.Mosaic_management(lccNormRaster, ' # 'mosaicRaster, "MINIMUM", "MATCH")') count = 0 while True: try: lu.write_log('Executing mosaic for link #'+str(linkId)) exec statement lu.write_log('Done with mosaic.') randomerror() except: count,tryAgain = lu.retry_arc_error(count,statement) lu.delete_data(mosaicRaster) lu.delete_dir(mosaicDir) # Try a new directory mosaicDir = path.join(cfg.LCCBASEDIR,'mos'+str(x+1)+ '_' + str(count)) lu.create_dir(mosaicDir) mosaicRaster = path.join(mosaicDir,mosFN) if not tryAgain: exec statement else: break endTime = time.clock() processTime = round((endTime - start_time), 2) if normalize == True: printText = "Normalized and mosaicked " else: printText = "Mosaicked NON-normalized " gprint(printText + "corridor for link ID #" + str(linkId) + " connecting core areas " + str(corex) + " and " + str(corey)+ " in " + str(processTime) + " seconds. " + str(int(linkCount)) + " out of " + str(int(numCorridorLinks)) + " links have been " "processed.") # temporarily disable links in linktable - don't want to mosaic # them twice for y in range (x+1,numLinks): corex1 = int(coreList[y,0]) corey1 = int(coreList[y,1]) if corex1 == corex and corey1 == corey: linkTable[y,cfg.LTB_LINKTYPE] = ( linkTable[y,cfg.LTB_LINKTYPE] + 1000) elif corex1==corey and corey1==corex: linkTable[y,cfg.LTB_LINKTYPE] = ( linkTable[y,cfg.LTB_LINKTYPE] + 1000) numGridsWritten = numGridsWritten + 1 if not SAVENORMLCCS: lu.delete_data(lccNormRaster) lu.delete_dir(clccdir) lu.create_dir(clccdir) else: if numGridsWritten == 100: # We only write up to 100 grids to any one folder # because otherwise Arc slows to a crawl dirCount = dirCount + 1 numGridsWritten = 0 clccdir = path.join(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM + str(dirCount)) gprint("Creating output folder: " + clccdir) gp.CreateFolder_management(cfg.LCCBASEDIR, path.basename(clccdir)) if numGridsWritten > 1 or dirCount > 0: lu.delete_data(lastMosaicRaster) lu.delete_dir(path.dirname(lastMosaicRaster)) lastMosaicRaster = mosaicRaster x = x + 1 #rows that were temporarily disabled rows = npy.where(linkTable[:,cfg.LTB_LINKTYPE]>1000) linkTable[rows,cfg.LTB_LINKTYPE] = ( linkTable[rows,cfg.LTB_LINKTYPE] - 1000) # --------------------------------------------------------------------- # Create output geodatabase if not gp.exists(outputGDB): gp.createfilegdb(cfg.OUTPUTDIR, path.basename(outputGDB)) if cfg.useArcpy: arcpy.env.workspace = outputGDB else: gp.workspace = outputGDB gp.pyramid = "NONE" gp.rasterstatistics = "NONE" # Copy mosaic raster to output geodatabase saveFloatRaster = False if saveFloatRaster == True: floatRaster = outputGDB + '\\' + PREFIX + mosaicBaseName + '_flt' # Full path statement = 'arcObj.CopyRaster_management(mosaicRaster, floatRaster)' try: exec statement except: pass # --------------------------------------------------------------------- # convert mosaic raster to integer intRaster = path.join(outputGDB,PREFIX + mosaicBaseName) if cfg.useArcpy: statement = ('outras = Int(Raster(mosaicRaster) - offset + 0.5); ' 'outras.save(intRaster)') else: expression = "int(" + mosaicRaster + " - " + str(offset) + " + 0.5)" statement = 'gp.SingleOutputMapAlgebra_sa(expression, intRaster)' count = 0 while True: try: exec statement randomerror() except: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec statement else: break # --------------------------------------------------------------------- if writeTruncRaster: # ----------------------------------------------------------------- # Set anything beyond cfg.CWDTHRESH to NODATA. if arcpyAvailable: cfg.useArcpy = True # For Alissa Pump's error with 10.1 cutoffText = str(cfg.CWDTHRESH) if cutoffText[-6:] == '000000': cutoffText = cutoffText[0:-6]+'m' elif cutoffText[-3:] == '000': cutoffText = cutoffText[0:-3]+'k' truncRaster = (outputGDB + '\\' + PREFIX + mosaicBaseName + '_truncated_at_' + cutoffText) count = 0 if cfg.useArcpy: statement = ('outRas = Raster(intRaster) * ' '(Con(Raster(intRaster) <= cfg.CWDTHRESH,1)); ' 'outRas.save(truncRaster)') else: expression = ("(" + intRaster + " * (con(" + intRaster + "<= " + str(cfg.CWDTHRESH) + ",1)))") statement = ('gp.SingleOutputMapAlgebra_sa(expression, ' 'truncRaster)') count = 0 while True: try: exec statement randomerror() except: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec statement else: break cfg.useArcpy = False # End fix for Alissa Pump's error with 10.1 # --------------------------------------------------------------------- # Check for unreasonably low minimum NLCC values try: mosaicGrid = path.join(cfg.LCCBASEDIR,'mos') # Copy to grid to test arcObj.CopyRaster_management(mosaicRaster, mosaicGrid) minObject = gp.GetRasterProperties(mosaicGrid, "MINIMUM") rasterMin = float(str(minObject.getoutput(0))) except: gp.AddWarning('\n------------------------------------------------') gp.AddWarning('WARNING: Raster minimum check failed in step 5. \n' 'This may mean the output rasters are corrupted. Please \n' 'be sure to check for valid rasters in '+ outputGDB) rasterMin = 0 tolerance = (float(gp.cellSize) * -10) if rasterMin < tolerance: lu.dashline(1) msg = ('WARNING: Minimum value of mosaicked corridor map is ' 'much less than zero ('+str(rasterMin)+').' '\nThis could mean that BOUNDING CIRCLE BUFFER DISTANCES ' 'were too small and a corridor passed outside of a ' 'bounding circle, or that a corridor passed outside of the ' 'resistance map. \n') gp.AddWarning(msg) gprint('\nWriting final LCP maps...') if cfg.STEP4: finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=4, thisStep=5) elif cfg.STEP3: finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=3, thisStep=5) else: # Don't know if step 4 was run, since this is started at step 5. # Use presence of previous linktable files to figure this out. # Linktable name includes step number. prevLinkTableFile = lu.get_prev_step_link_table(step=5) prevStepInd = len(prevLinkTableFile) - 5 lastStep = prevLinkTableFile[prevStepInd] finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep, thisStep=5) outlinkTableFile = lu.get_this_step_link_table(step=5) gprint('Updating ' + outlinkTableFile) lu.write_link_table(linkTable, outlinkTableFile) linkTableLogFile = path.join(cfg.LOGDIR, "linkTable_s5.csv") lu.write_link_table(linkTable, linkTableLogFile) linkTableFinalFile = path.join(cfg.OUTPUTDIR, PREFIX + "_linkTable_s5.csv") lu.write_link_table(finalLinkTable, linkTableFinalFile) gprint('Copy of final linkTable written to '+ linkTableFinalFile) gprint('Creating shapefiles with linework for links.') try: lu.write_link_maps(outlinkTableFile, step=5) except: lu.write_link_maps(outlinkTableFile, step=5) # Create final linkmap files in output directory, and remove files from # scratch. lu.copy_final_link_maps(step=5) if not SAVENORMLCCS: lu.delete_dir(cfg.LCCBASEDIR) # Build statistics for corridor rasters gp.addmessage('\nBuilding output statistics and pyramids ' 'for corridor raster') lu.build_stats(intRaster) if writeTruncRaster: gp.addmessage('Building output statistics ' 'for truncated corridor raster') lu.build_stats(truncRaster) # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.dashline(1) gprint('****Failed in step 5. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 5. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME) return
def STEP8_calc_pinchpoints(): """ Maps pinch points in Linkage Mapper corridors using Circuitscape given CWD calculations from s3_calcCwds.py. """ try: lu.dashline(0) gprint('Running script ' + _SCRIPT_NAME) restartFlag = False if cfg.CWDCUTOFF < 0: cfg.CWDCUTOFF = cfg.CWDCUTOFF * -1 restartFlag = True # Restart code in progress CSPATH = lu.get_cs_path() outputGDB = path.join(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) arcpy.OverWriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.pyramid = "NONE" arcpy.env.rasterstatistics = "NONE" # set the analysis extent and cell size to that of the resistance # surface arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.snapraster = cfg.RESRAST resRaster = cfg.RESRAST arcpy.env.extent = "MINOF" minObject = arcpy.GetRasterProperties_management(resRaster, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) if rasterMin <= 0: msg = ( 'Error: resistance raster cannot have 0 or negative values.') lu.raise_error(msg) if cfg.DO_ADJACENTPAIRS: prevLcpShapefile = lu.get_lcp_shapefile(None, thisStep=8) if not arcpy.Exists(prevLcpShapefile): msg = ('Cannot find an LCP shapefile from step 5. Please ' 'rerun that step and any previous ones if necessary.') lu.raise_error(msg) # Remove lcp shapefile lcpShapefile = path.join(cfg.DATAPASSDIR, "lcpLines_s8.shp") lu.delete_data(lcpShapefile) inLinkTableFile = lu.get_prev_step_link_table(step=8) linkTable = lu.load_link_table(inLinkTableFile) numLinks = linkTable.shape[0] numCorridorLinks = lu.report_links(linkTable) if numCorridorLinks == 0: lu.dashline(1) msg = ('\nThere are no linkages. Bailing.') lu.raise_error(msg) if linkTable.shape[1] < 16: # If linktable has no entries from prior # centrality or pinchpint analyses extraCols = npy.zeros((numLinks, 6), dtype="float64") linkTable = linkTable[:, 0:10] linkTable = npy.append(linkTable, extraCols, axis=1) linkTable[:, cfg.LTB_LCPLEN] = -1 linkTable[:, cfg.LTB_CWDEUCR] = -1 linkTable[:, cfg.LTB_CWDPATHR] = -1 linkTable[:, cfg.LTB_EFFRESIST] = -1 linkTable[:, cfg.LTB_CWDTORR] = -1 linkTable[:, cfg.LTB_CURRENT] = -1 del extraCols # set up directories for circuit and circuit mosaic grids # Create output geodatabase if not arcpy.Exists(cfg.PINCHGDB): arcpy.CreateFileGDB_management(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) mosaicRaster = path.join(cfg.CIRCUITBASEDIR, "current_mos" + tif) coresToProcess = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) maxCoreNum = max(coresToProcess) del coresToProcess lu.dashline(0) coreList = linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1] coreList = npy.sort(coreList) #gprint('There are ' + str(len(npy.unique(coreList))) ' core areas.') INCIRCUITDIR = cfg.CIRCUITBASEDIR OUTCIRCUITDIR = path.join(cfg.CIRCUITBASEDIR, cfg.CIRCUITOUTPUTDIR_NM) CONFIGDIR = path.join(INCIRCUITDIR, cfg.CIRCUITCONFIGDIR_NM) # Cutoff value text to append to filenames cutoffText = str(cfg.CWDCUTOFF) if cutoffText[-6:] == '000000': cutoffText = cutoffText[0:-6] + 'm' elif cutoffText[-3:] == '000': cutoffText = cutoffText[0:-3] + 'k' if cfg.SQUARERESISTANCES: # Square resistance values squaredRaster = path.join(cfg.SCRATCHDIR, 'res_sqr') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR outRas = Raster(resRaster) * Raster(resRaster) outRas.save(squaredRaster) resRaster = squaredRaster if cfg.DO_ADJACENTPAIRS: linkLoop = 0 lu.dashline(1) gprint('Mapping pinch points in individual corridors \n' 'using Circuitscape.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(2) for x in range(0, numLinks): linkId = str(int(linkTable[x, cfg.LTB_LINKID])) if not (linkTable[x, cfg.LTB_LINKTYPE] > 0): continue linkLoop = linkLoop + 1 linkDir = path.join(cfg.SCRATCHDIR, 'link' + linkId) if restartFlag == True and path.exists(linkDir): gprint('continuing') continue restartFlag = False lu.create_dir(linkDir) start_time1 = time.clock() # source and target cores corex = int(coreList[x, 0]) corey = int(coreList[x, 1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) lccNormRaster = path.join(linkDir, 'lcc_norm') arcpy.env.extent = "MINOF" link = lu.get_links_from_core_pairs(linkTable, corex, corey) lcDist = float(linkTable[link, cfg.LTB_CWDIST]) # Normalized lcc rasters are created by adding cwd rasters # and subtracting the least cost distance between them. outRas = Raster(cwdRaster1) + Raster(cwdRaster2) - lcDist outRas.save(lccNormRaster) #create raster mask resMaskRaster = path.join(linkDir, 'res_mask' + tif) #create raster mask outCon = arcpy.sa.Con( Raster(lccNormRaster) <= cfg.CWDCUTOFF, 1) outCon.save(resMaskRaster) # Convert to poly. Use as mask to clip resistance raster. resMaskPoly = path.join(linkDir, 'res_mask_poly.shp') arcpy.RasterToPolygon_conversion(resMaskRaster, resMaskPoly, "NO_SIMPLIFY") arcpy.env.extent = resMaskPoly # Includes 0 values in some cases with CP LI model if tif # so using ESRI Grid format resClipRasterMasked = path.join(linkDir, 'res_clip_m') # Extract masked resistance raster. # Needs to be float to get export to npy to work. outRas = arcpy.sa.ExtractByMask(resRaster, resMaskPoly) + 0.0 outRas.save(resClipRasterMasked) resNpyFN = 'resistances_link_' + linkId + '.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy( resClipRasterMasked, resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) lu.warn('Warning:') lu.warn('Circuitscape can only solve 2-3 million nodes') lu.warn( 'per gigabyte of available RAM. \nTotal physical RAM' ' on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~' + str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(2) corePairRaster = path.join(linkDir, 'core_pairs' + tif) arcpy.env.extent = resClipRasterMasked # Next result needs to be floating pt for numpy export outCon = arcpy.sa.Con( Raster(cwdRaster1) == 0, corex, arcpy.sa.Con(Raster(cwdRaster2) == 0, corey + 0.0)) outCon.save(corePairRaster) coreNpyFN = 'cores_link_' + linkId + '.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy( corePairRaster, coreNpyFile) arcpy.env.extent = "MINOF" # Set circuitscape options and call options = lu.setCircuitscapeOptions() if cfg.WRITE_VOLT_MAPS == True: options['write_volt_maps'] = True options['habitat_file'] = resNpyFile # if int(linkId) > 2: # options['habitat_file'] = 'c:\\test.dummy' options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero'] = True outputFN = 'Circuitscape_link' + linkId + '.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) if numElements > 250000: options['print_timings'] = True configFN = 'pinchpoint_config' + linkId + '.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('Processing link ID #' + str(linkId) + '. Resistance map' ' has ' + str(int(numResistanceNodes)) + ' nodes.') memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): print_failure(numResistanceNodes, memFlag, 10) numElements, numNodes = export_ras_to_npy( resClipRasterMasked, resNpyFile) memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): msg = ( '\nCircuitscape failed. See error information above.') arcpy.AddError(msg) lu.write_log(msg) exit(1) # Either set core areas to nodata in current map or # divide each by its radius currentRaster = path.join(linkDir, "current" + tif) import_npy_to_ras(currentMap, corePairRaster, currentRaster) if cfg.WRITE_VOLT_MAPS == True: voltFN = ('Circuitscape_link' + linkId + '_voltmap_' + str(corex) + '_' + str(corey) + '.npy') voltMap = path.join(OUTCIRCUITDIR, voltFN) voltRaster = path.join( outputGDB, cfg.PREFIX + "_voltMap_" + str(corex) + '_' + str(corey)) import_npy_to_ras(voltMap, corePairRaster, voltRaster) gprint('Building output statistics and pyramids ' 'for voltage raster\n') lu.build_stats(voltRaster) arcpy.env.extent = currentRaster if SETCORESTONULL: # Set core areas to NoData in current map for color ramping currentRaster2 = currentRaster + '2' + tif outCon = arcpy.sa.Con( arcpy.sa.IsNull(Raster(corePairRaster)), Raster(currentRaster)) outCon.save(currentRaster2) currentRaster = currentRaster2 arcpy.env.extent = "MAXOF" if linkLoop == 1: lu.delete_data(mosaicRaster) @retry(10) def copyRas2(): arcpy.CopyRaster_management(currentRaster, mosaicRaster) copyRas2() else: @retry(10) def mosaicRas(): arcpy.Mosaic_management(currentRaster, mosaicRaster, "MAXIMUM", "MATCH") mosaicRas() resistancesFN = ('Circuitscape_link' + linkId + '_resistances_3columns.out') resistancesFile = path.join(OUTCIRCUITDIR, resistancesFN) resistances = npy.loadtxt(resistancesFile, dtype='Float64', comments='#') resistance = float(str(arcpy.env.cellSize)) * resistances[2] linkTable[link, cfg.LTB_EFFRESIST] = resistance # Ratio if not cfg.SQUARERESISTANCES: linkTable[link, cfg.LTB_CWDTORR] = ( linkTable[link, cfg.LTB_CWDIST] / linkTable[link, cfg.LTB_EFFRESIST]) # Clean up if cfg.SAVE_TEMP_CIRCUIT_FILES == False: lu.delete_file(coreNpyFile) coreNpyBase, extension = path.splitext(coreNpyFile) lu.delete_data(coreNpyBase + '.hdr') lu.delete_file(resNpyFile) resNpyBase, extension = path.splitext(resNpyFile) lu.delete_data(resNpyBase + '.hdr') lu.delete_file(currentMap) curMapBase, extension = path.splitext(currentMap) lu.delete_data(curMapBase + '.hdr') lu.delete_data(currentRaster) lu.clean_out_workspace(linkDir) lu.delete_dir(linkDir) gprint('Finished with link ID #' + str(linkId) + '. ' + str(linkLoop) + ' out of ' + str(numCorridorLinks) + ' links have been processed.') start_time1 = lu.elapsed_time(start_time1) outputRaster = path.join( outputGDB, cfg.PREFIX + "_current_adjacentPairs_" + cutoffText) lu.delete_data(outputRaster) @retry(10) def copyRas(): arcpy.CopyRaster_management(mosaicRaster, outputRaster) copyRas() gprint('Building output statistics and pyramids ' 'for corridor pinch point raster\n') lu.build_stats(outputRaster) finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=5, thisStep=8) linkTableFile = path.join(cfg.DATAPASSDIR, "linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFile, inLinkTableFile) linkTableFinalFile = path.join( cfg.OUTPUTDIR, cfg.PREFIX + "_linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFinalFile, inLinkTableFile) gprint('Copy of linkTable written to ' + linkTableFinalFile) #fixme: update sticks? gprint('Creating shapefiles with linework for links.') lu.write_link_maps(linkTableFinalFile, step=8) # Copy final link maps to gdb. lu.copy_final_link_maps(step=8) lu.delete_data(mosaicRaster) if not cfg.DO_ALLPAIRS: # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) return lu.dashline(1) gprint('Mapping global pinch points among all\n' 'core area pairs using Circuitscape.') if cfg.ALL_PAIR_SCENARIO == 'pairwise': gprint('Circuitscape will be run in PAIRWISE mode.') else: gprint('Circuitscape will be run in ALL-TO-ONE mode.') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST S8CORE_RAS = "s8core_ras" s8CoreRasPath = path.join(cfg.SCRATCHDIR, S8CORE_RAS) arcpy.FeatureToRaster_conversion(cfg.COREFC, cfg.COREFN, s8CoreRasPath, arcpy.env.cellSize) binaryCoreRaster = path.join(cfg.SCRATCHDIR, "core_ras_bin") # The following commands cause file lock problems on save. using gp # instead. # outCon = arcpy.sa.Con(S8CORE_RAS, 1, "#", "VALUE > 0") # outCon.save(binaryCoreRaster) # gp.Con_sa(s8CoreRasPath, 1, binaryCoreRaster, "#", "VALUE > 0") outCon = arcpy.sa.Con(Raster(s8CoreRasPath) > 0, 1) outCon.save(binaryCoreRaster) s5corridorRas = path.join(cfg.OUTPUTGDB, cfg.PREFIX + "_corridors") if not arcpy.Exists(s5corridorRas): s5corridorRas = path.join(cfg.OUTPUTGDB, cfg.PREFIX + "_lcc_mosaic_int") outCon = arcpy.sa.Con( Raster(s5corridorRas) <= cfg.CWDCUTOFF, Raster(resRaster), arcpy.sa.Con(Raster(binaryCoreRaster) > 0, Raster(resRaster))) resRasClipPath = path.join(cfg.SCRATCHDIR, 'res_ras_clip') outCon.save(resRasClipPath) arcpy.env.cellSize = resRasClipPath arcpy.env.extent = resRasClipPath s8CoreRasClipped = s8CoreRasPath + '_c' # Produce core raster with same extent as clipped resistance raster # added to ensure correct data type- nodata values were positive for # cores otherwise outCon = arcpy.sa.Con(arcpy.sa.IsNull(Raster(s8CoreRasPath)), -9999, Raster(s8CoreRasPath)) outCon.save(s8CoreRasClipped) resNpyFN = 'resistances.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy( resRasClipPath, resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) lu.warn('Warning:') lu.warn('Circuitscape can only solve 2-3 million nodes') lu.warn('per gigabyte of available RAM. \nTotal physical RAM ' 'on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~' + str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(0) coreNpyFN = 'cores.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy(s8CoreRasClipped, coreNpyFile) arcpy.env.extent = "MINOF" options = lu.setCircuitscapeOptions() options['scenario'] = cfg.ALL_PAIR_SCENARIO options['habitat_file'] = resNpyFile options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero'] = True outputFN = 'Circuitscape.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) options['print_timings'] = True configFN = 'pinchpoint_allpair_config.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('\nResistance map has ' + str(int(numResistanceNodes)) + ' nodes.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(0) call_circuitscape(CSPATH, outConfigFile) # test = subprocess.call([CSPATH, outConfigFile], # creationflags = subprocess.CREATE_NEW_CONSOLE) if options['scenario'] == 'pairwise': rasterSuffix = "_current_allPairs_" + cutoffText else: rasterSuffix = "_current_allToOne_" + cutoffText currentFN = 'Circuitscape_cum_curmap.npy' currentMap = path.join(OUTCIRCUITDIR, currentFN) outputRaster = path.join(outputGDB, cfg.PREFIX + rasterSuffix) currentRaster = path.join(cfg.SCRATCHDIR, "current") try: import_npy_to_ras(currentMap, resRasClipPath, outputRaster) except: lu.dashline(1) msg = ('ERROR: Circuitscape failed. \n' 'Note: Circuitscape can only solve 2-3 million nodes' '\nper gigabyte of available RAM. The resistance ' '\nraster for the last corridor had ' + str(numResistanceNodes) + ' nodes.\n\nResistance ' 'raster values that vary by >6 orders of \nmagnitude' ' can also cause failures, as can a mismatch in ' '\ncore area and resistance raster extents.') arcpy.AddError(msg) lu.write_log(msg) exit(1) #set core areas to nodata if SETCORESTONULL: # Set core areas to NoData in current map for color ramping outputRasterND = outputRaster + '_noDataCores' outCon = arcpy.sa.SetNull( Raster(s8CoreRasClipped) > 0, Raster(outputRaster)) outCon.save(outputRasterND) gprint('\nBuilding output statistics and pyramids ' 'for centrality raster.') lu.build_stats(outputRaster) lu.build_stats(outputRasterND) # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) # Return GEOPROCESSING specific errors except arcpy.ExecuteError: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME)
def STEP3_calc_cwds(): """Calculates cost-weighted distances from each core area. Uses bounding circles around source and target cores to limit extent of cwd calculations and speed computation. """ try: lu.dashline(1) gprint('Running script ' + _SCRIPT_NAME) lu.dashline(0) # Super secret setting to re-start failed run. Enter 'RESTART' as the # Name of the pairwise distance table in step 2, and uncheck step 2. # We can eventually place this in a .ini file. rerun = False if cfg.S2EUCDISTFILE != None: if cfg.S2EUCDISTFILE.lower() == "restart": rerun = True # if cfg.TMAXCWDIST is None: # gprint('NOT using a maximum cost-weighted distance.') # else: # gprint('Max cost-weighted distance for CWD calcs set ' # 'to ' + str(cfg.TMAXCWDIST) + '\n') if (cfg.BUFFERDIST) is not None: gprint('Bounding circles plus a buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units will ' 'be used \n to limit extent of cost distance ' 'calculations.') elif cfg.TOOL <> cfg.TOOL_CC: gprint('NOT using bounding circles in cost distance ' 'calculations.') # set the analysis extent and cell size # So we don't extract rasters that go beyond extent of original raster if arcpy: arcpy.env.cellSize = cfg.RESRAST arcpy.env.extent="MINOF" else: gp.cellSize = gp.Describe(cfg.RESRAST).MeanCellHeight gp.Extent = "MINOF" gp.mask = cfg.RESRAST if arcpy: arcpy.env.overwriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR else: gp.OverwriteOutput = True gp.workspace = cfg.SCRATCHDIR gp.scratchWorkspace = cfg.ARCSCRATCHDIR # Load linkTable (created in previous script) linkTableFile = lu.get_prev_step_link_table(step=3) linkTable = lu.load_link_table(linkTableFile) lu.report_links(linkTable) # Identify cores to map from LinkTable coresToMap = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) numCoresToMap = len(coresToMap) if numCoresToMap < 3: # No need to check for intermediate cores, because there aren't any cfg.S3DROPLCCSic = False else: cfg.S3DROPLCCSic = cfg.S3DROPLCCS gprint('\nNumber of core areas to connect: ' + str(numCoresToMap)) if rerun: # If picking up a failed run, make sure needed files are there lu.dashline(1) gprint ('\n****** RESTART MODE ENABLED ******\n') gprint ('**** NOTE: This mode picks up step 3 where a\n' 'previous run left off due to a crash or user\n' 'abort. It assumes you are using the same input\n' 'data used in the terminated run.****\n') lu.dashline(0) lu.snooze(10) savedLinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") coreListFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") if not path.exists(savedLinkTableFile) or not path.exists( coreListFile): gprint('No partial results file found from previous ' 'stopped run. Starting run from beginning.\n') lu.dashline(0) rerun = False # If picking up a failed run, use old folders if not rerun: startIndex = 0 if cfg.TOOL <> cfg.TOOL_CC: lu.make_cwd_paths(max(coresToMap)) # Set up cwd directories # make a feature layer for input cores to select from gp.MakeFeatureLayer(cfg.COREFC, cfg.FCORES) # Drop links that are too long gprint('\nChecking for corridors that are too long to map.') DISABLE_LEAST_COST_NO_VAL = False linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, 0, cfg.MAXCOSTDIST, 0, DISABLE_LEAST_COST_NO_VAL) # ------------------------------------------------------------------ # Bounding boxes if (cfg.BUFFERDIST) is not None: # create bounding boxes around cores start_time = time.clock() # lu.dashline(1) gprint('Calculating bounding boxes for core areas.') extentBoxList = npy.zeros((0,5), dtype='float32') for x in range(len(coresToMap)): core = coresToMap[x] boxCoords = lu.get_extent_box_coords(core) extentBoxList = npy.append(extentBoxList, boxCoords, axis=0) gprint('\nDone calculating bounding boxes.') start_time = lu.elapsed_time(start_time) # lu.dashline() # Bounding circle code if cfg.BUFFERDIST is not None: # Make a set of circles encompassing core areas we'll be connecting start_time = time.clock() gprint('Calculating bounding circles around potential' ' corridors.') # x y corex corey radius- stores data for bounding circle centroids boundingCirclePointArray = npy.zeros((0,5), dtype='float32') circleList = npy.zeros((0,3), dtype='int32') numLinks = linkTable.shape[0] for x in range(0, numLinks): if ((linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_CORR) or (linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_KEEP)): # if it's a valid corridor link linkId = int(linkTable[x,cfg.LTB_LINKID]) # fixme- this code is clumsy- can trim down cores = npy.zeros((1,3), dtype='int32') cores[0,:] = npy.sort([0, linkTable[x,cfg.LTB_CORE1], linkTable[x,cfg.LTB_CORE2]]) corex = cores[0,1] corey = cores[0,2] cores[0,0] = linkId ################### foundFlag = False for y in range(0,len(circleList)): # clumsy if (circleList[y,1] == corex and circleList[y,2] == corey): foundFlag = True if not foundFlag: circlePointData = ( lu.get_bounding_circle_data(extentBoxList, corex, corey, cfg.BUFFERDIST)) boundingCirclePointArray = ( npy.append(boundingCirclePointArray, circlePointData, axis=0)) # keep track of which cores we draw bounding circles # around circleList = npy.append(circleList, cores, axis=0) gprint('\nCreating bounding circles using buffer ' 'analysis.') dir, BNDCIRCENS = path.split(cfg.BNDCIRCENS) lu.make_points(cfg.SCRATCHDIR, boundingCirclePointArray, BNDCIRCENS) lu.delete_data(cfg.BNDCIRS) gp.buffer_analysis(cfg.BNDCIRCENS, cfg.BNDCIRS, "radius") gp.deletefield (cfg.BNDCIRS, "BUFF_DIST") gprint('Successfully created bounding circles around ' 'potential corridors using \na buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.') start_time = lu.elapsed_time(start_time) gprint('Reducing global processing area using bounding ' 'circle plus buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.\n') extentBoxList = npy.zeros((0,5),dtype='float32') boxCoords = lu.get_extent_box_coords() extentBoxList = npy.append(extentBoxList,boxCoords,axis=0) extentBoxList[0,0] = 0 boundingCirclePointArray = npy.zeros((0,5),dtype='float32') circlePointData=lu.get_bounding_circle_data(extentBoxList, 0, 0, cfg.BUFFERDIST) dir, BNDCIRCEN = path.split(cfg.BNDCIRCEN) lu.make_points(cfg.SCRATCHDIR, circlePointData, BNDCIRCEN) lu.delete_data(cfg.BNDCIR) gp.buffer_analysis(cfg.BNDCIRCEN, cfg.BNDCIR, "radius") gprint('Extracting raster....') cfg.BOUNDRESIS = cfg.BOUNDRESIS + tif lu.delete_data(cfg.BOUNDRESIS) count = 0 statement = ( 'gp.ExtractByMask_sa(cfg.RESRAST, cfg.BNDCIR, cfg.BOUNDRESIS)') while True: try: exec statement randomerror() except: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec statement else: break gprint('\nReduced resistance raster extracted using ' 'bounding circle.') else: #if not using bounding circles, just go with resistance raster. cfg.BOUNDRESIS = cfg.RESRAST # --------------------------------------------------------------------- # Rasterize core areas to speed cost distance calcs # lu.dashline(1) gprint("Creating core area raster.") gp.SelectLayerByAttribute(cfg.FCORES, "CLEAR_SELECTION") if arcpy: arcpy.env.cellSize = cfg.BOUNDRESIS arcpy.env.extent = cfg.BOUNDRESIS else: gp.cellSize = gp.Describe(cfg.BOUNDRESIS).MeanCellHeight gp.extent = gp.Describe(cfg.BOUNDRESIS).extent if rerun: # saved linktable replaces the one now in memory linkTable = lu.load_link_table(savedLinkTableFile) coresToMapSaved = npy.loadtxt(coreListFile, dtype='Float64', comments='#', delimiter=',') startIndex = coresToMapSaved[0] # Index of core where we left off del coresToMapSaved gprint ('\n****** Re-starting run at core area number ' + str(int(coresToMap[startIndex]))+ ' ******\n') lu.dashline(0) if arcpy: arcpy.env.extent = "MINOF" else: gp.extent = "MINOF" #---------------------------------------------------------------------- # Loop through cores, do cwd calcs for each if cfg.TOOL == cfg.TOOL_CC: gprint("\nMapping least-cost paths.\n") else: gprint("\nStarting cost distance calculations.\n") lcpLoop = 0 failures = 0 x = startIndex endIndex = len(coresToMap) linkTableMod = linkTable.copy() while x < endIndex: startTime1 = time.clock() # Modification of linkTable in function was causing problems. so # make a copy: linkTablePassed = linkTableMod.copy() (linkTableReturned, failures, lcpLoop) = do_cwd_calcs(x, linkTablePassed, coresToMap, lcpLoop, failures) if failures == 0: # If iteration was successful, continue with next core linkTableMod = linkTableReturned sourceCore = int(coresToMap[x]) gprint('Done with all calculations for core ID #' + str(sourceCore) + '. ' + str(int(x + 1)) + ' of ' + str(endIndex) + ' cores have been processed.') start_time = lu.elapsed_time(startTime1) outlinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.write_link_table(linkTableMod, outlinkTableFile) # Increment loop counter x = x + 1 else: # If iteration failed, try again after a wait period delay_restart(failures) #---------------------------------------------------------------------- linkTable = linkTableMod # reinstate temporarily disabled links rows = npy.where(linkTable[:,cfg.LTB_LINKTYPE] > 1000) linkTable[rows,cfg.LTB_LINKTYPE] = (linkTable[rows,cfg.LTB_LINKTYPE] - 1000) # Drop links that are too long DISABLE_LEAST_COST_NO_VAL = True linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, cfg.MINEUCDIST, cfg.MAXCOSTDIST, cfg.MINCOSTDIST, DISABLE_LEAST_COST_NO_VAL) # Write link table file outlinkTableFile = lu.get_this_step_link_table(step=3) gprint('Updating ' + outlinkTableFile) lu.write_link_table(linkTable, outlinkTableFile) linkTableLogFile = path.join(cfg.LOGDIR, "linkTable_s3.csv") lu.write_link_table(linkTable, linkTableLogFile) start_time = time.clock() gprint('Creating shapefiles with linework for links...') try: lu.write_link_maps(outlinkTableFile, step=3) except: lu.write_link_maps(outlinkTableFile, step=3) start_time = lu.elapsed_time(start_time) gprint('\nIndividual cost-weighted distance layers written ' 'to "cwd" directory. \n') gprint(outlinkTableFile + '\n updated with cost-weighted distances between core areas.') #Clean up temporary files for restart code tempFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") lu.delete_file(tempFile) tempFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.delete_file(tempFile) # Check if climate tool is calling linkage mapper if cfg.TOOL == cfg.TOOL_CC: coreList = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) for core in coreList: cwdRaster = lu.get_cwd_path(int(core)) back_rast = cwdRaster.replace("cwd_", "back_") lu.delete_data(back_rast) # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME) return
def calc_lccs(normalize): try: if normalize: mosaicBaseName = "_corridors" writeTruncRaster = cfg.WRITETRUNCRASTER outputGDB = cfg.OUTPUTGDB if cfg.CALCNONNORMLCCS: SAVENORMLCCS = False else: SAVENORMLCCS = cfg.SAVENORMLCCS else: mosaicBaseName = "_NON_NORMALIZED_corridors" SAVENORMLCCS = False outputGDB = cfg.EXTRAGDB writeTruncRaster = False lu.dashline(1) gprint('Running script ' + _SCRIPT_NAME) linkTableFile = lu.get_prev_step_link_table(step=5) if cfg.useArcpy: arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.overwriteOutput = True arcpy.env.compression = "NONE" else: gp.workspace = cfg.SCRATCHDIR gp.scratchWorkspace = cfg.ARCSCRATCHDIR gp.OverwriteOutput = True if cfg.MAXEUCDIST is not None: gprint('Max Euclidean distance between cores') gprint('for linkage mapping set to ' + str(cfg.MAXEUCDIST)) if cfg.MAXCOSTDIST is not None: gprint('Max cost-weighted distance between cores') gprint('for linkage mapping set to ' + str(cfg.MAXCOSTDIST)) # set the analysis extent and cell size to that of the resistance # surface if cfg.useArcpy: arcpy.env.Extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.env.snapRaster = cfg.RESRAST arcpy.env.mask = cfg.RESRAST else: gp.Extent = (gp.Describe(cfg.RESRAST)).Extent gp.cellSize = gp.Describe(cfg.RESRAST).MeanCellHeight gp.mask = cfg.RESRAST gp.snapraster = cfg.RESRAST linkTable = lu.load_link_table(linkTableFile) numLinks = linkTable.shape[0] numCorridorLinks = lu.report_links(linkTable) if numCorridorLinks == 0: lu.dashline(1) msg = ('\nThere are no corridors to map. Bailing.') lu.raise_error(msg) if not cfg.STEP3 and not cfg.STEP4: # re-check for links that are too long or in case script run out of # sequence with more stringent settings gprint('Double-checking for corridors that are too long to map.') DISABLE_LEAST_COST_NO_VAL = True linkTable, numDroppedLinks = lu.drop_links( linkTable, cfg.MAXEUCDIST, cfg.MINEUCDIST, cfg.MAXCOSTDIST, cfg.MINCOSTDIST, DISABLE_LEAST_COST_NO_VAL) # Added to try to speed up: gp.pyramid = "NONE" gp.rasterstatistics = "NONE" # set up directories for normalized lcc and mosaic grids dirCount = 0 gprint("Creating output folder: " + cfg.LCCBASEDIR) lu.delete_dir(cfg.LCCBASEDIR) gp.CreateFolder_management(path.dirname(cfg.LCCBASEDIR), path.basename(cfg.LCCBASEDIR)) gp.CreateFolder_management(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM) clccdir = path.join(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM) # mosaicGDB = path.join(cfg.LCCBASEDIR, "mosaic.gdb") # gp.createfilegdb(cfg.LCCBASEDIR, "mosaic.gdb") #mosaicRaster = mosaicGDB + '\\' + "nlcc_mos" # Full path gprint("") if normalize: gprint('Normalized least-cost corridors will be written ' 'to ' + clccdir + '\n') PREFIX = cfg.PREFIX # Add CWD layers for core area pairs to produce NORMALIZED LCC layers numGridsWritten = 0 coreList = linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1] coreList = npy.sort(coreList) x = 0 linkCount = 0 endIndex = numLinks while x < endIndex: if (linkTable[x, cfg.LTB_LINKTYPE] < 1): # If not a valid link x = x + 1 continue linkCount = linkCount + 1 start_time = time.clock() linkId = str(int(linkTable[x, cfg.LTB_LINKID])) # source and target cores corex = int(coreList[x, 0]) corey = int(coreList[x, 1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) if not gp.Exists(cwdRaster1): msg = ('\nError: cannot find cwd raster:\n' + cwdRaster1) if not gp.Exists(cwdRaster2): msg = ('\nError: cannot find cwd raster:\n' + cwdRaster2) lu.raise_error(msg) lccNormRaster = path.join(clccdir, str(corex) + "_" + str(corey)) # + ".tif") if cfg.useArcpy: arcpy.env.Extent = "MINOF" else: gp.Extent = "MINOF" # FIXME: need to check for this?: # if exists already, don't re-create #if not gp.Exists(lccRaster): link = lu.get_links_from_core_pairs(linkTable, corex, corey) offset = 10000 # Normalized lcc rasters are created by adding cwd rasters and # subtracting the least cost distance between them. count = 0 if arcpyAvailable: cfg.useArcpy = True # Fixes Canran Liu's bug with lcDist if cfg.useArcpy: lcDist = (float(linkTable[link, cfg.LTB_CWDIST]) - offset) if normalize: statement = ( 'outras = Raster(cwdRaster1) + Raster(' 'cwdRaster2) - lcDist; outras.save(lccNormRaster)') else: statement = ('outras =Raster(cwdRaster1) + Raster(' 'cwdRaster2); outras.save(lccNormRaster)') else: if normalize: lcDist = str(linkTable[link, cfg.LTB_CWDIST] - offset) expression = (cwdRaster1 + " + " + cwdRaster2 + " - " + lcDist) else: expression = (cwdRaster1 + " + " + cwdRaster2) statement = ('gp.SingleOutputMapAlgebra_sa(expression, ' 'lccNormRaster)') count = 0 while True: try: exec statement randomerror() except: count, tryAgain = lu.retry_arc_error(count, statement) if not tryAgain: exec statement else: break cfg.useArcpy = False # End fix for Conran Liu's bug with lcDist if normalize and cfg.useArcpy: try: minObject = gp.GetRasterProperties(lccNormRaster, "MINIMUM") rasterMin = float(str(minObject.getoutput(0))) except: gp.AddWarning( '\n------------------------------------------------') gp.AddWarning( 'WARNING: Raster minimum check failed in step 5. \n' 'This may mean the output rasters are corrupted. Please \n' 'be sure to check for valid rasters in ' + outputGDB) rasterMin = 0 tolerance = (float(gp.cellSize) * -10) + offset if rasterMin < tolerance: lu.dashline(1) msg = ( 'WARNING: Minimum value of a corridor #' + str(x + 1) + ' is much less than zero (' + str(rasterMin) + ').' '\nThis could mean that BOUNDING CIRCLE BUFFER DISTANCES ' 'were too small and a corridor passed outside of a ' 'bounding circle, or that a corridor passed outside of the ' 'resistance map. \n') gp.AddWarning(msg) if cfg.useArcpy: arcpy.env.Extent = cfg.RESRAST else: gp.Extent = (gp.Describe(cfg.RESRAST)).Extent mosaicDir = path.join(cfg.LCCBASEDIR, 'mos' + str(x + 1)) lu.create_dir(mosaicDir) mosFN = 'mos' #.tif' change and move mosaicRaster = path.join(mosaicDir, mosFN) if numGridsWritten == 0 and dirCount == 0: #If this is the first grid then copy rather than mosaic arcObj.CopyRaster_management(lccNormRaster, mosaicRaster) else: rasterString = '"' + lccNormRaster + ";" + lastMosaicRaster + '"' statement = ('arcObj.MosaicToNewRaster_management(' 'rasterString,mosaicDir,mosFN, "", ' '"32_BIT_FLOAT", gp.cellSize, "1", "MINIMUM", ' '"MATCH")') # statement = ('arcpy.Mosaic_management(lccNormRaster, ' # 'mosaicRaster, "MINIMUM", "MATCH")') count = 0 while True: try: lu.write_log('Executing mosaic for link #' + str(linkId)) exec statement lu.write_log('Done with mosaic.') randomerror() except: count, tryAgain = lu.retry_arc_error(count, statement) lu.delete_data(mosaicRaster) lu.delete_dir(mosaicDir) # Try a new directory mosaicDir = path.join( cfg.LCCBASEDIR, 'mos' + str(x + 1) + '_' + str(count)) lu.create_dir(mosaicDir) mosaicRaster = path.join(mosaicDir, mosFN) if not tryAgain: exec statement else: break endTime = time.clock() processTime = round((endTime - start_time), 2) if normalize == True: printText = "Normalized and mosaicked " else: printText = "Mosaicked NON-normalized " gprint(printText + "corridor for link ID #" + str(linkId) + " connecting core areas " + str(corex) + " and " + str(corey) + " in " + str(processTime) + " seconds. " + str(int(linkCount)) + " out of " + str(int(numCorridorLinks)) + " links have been " "processed.") # temporarily disable links in linktable - don't want to mosaic # them twice for y in range(x + 1, numLinks): corex1 = int(coreList[y, 0]) corey1 = int(coreList[y, 1]) if corex1 == corex and corey1 == corey: linkTable[y, cfg.LTB_LINKTYPE] = ( linkTable[y, cfg.LTB_LINKTYPE] + 1000) elif corex1 == corey and corey1 == corex: linkTable[y, cfg.LTB_LINKTYPE] = ( linkTable[y, cfg.LTB_LINKTYPE] + 1000) numGridsWritten = numGridsWritten + 1 if not SAVENORMLCCS: lu.delete_data(lccNormRaster) lu.delete_dir(clccdir) lu.create_dir(clccdir) else: if numGridsWritten == 100: # We only write up to 100 grids to any one folder # because otherwise Arc slows to a crawl dirCount = dirCount + 1 numGridsWritten = 0 clccdir = path.join(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM + str(dirCount)) gprint("Creating output folder: " + clccdir) gp.CreateFolder_management(cfg.LCCBASEDIR, path.basename(clccdir)) if numGridsWritten > 1 or dirCount > 0: lu.delete_data(lastMosaicRaster) lu.delete_dir(path.dirname(lastMosaicRaster)) lastMosaicRaster = mosaicRaster x = x + 1 #rows that were temporarily disabled rows = npy.where(linkTable[:, cfg.LTB_LINKTYPE] > 1000) linkTable[rows, cfg.LTB_LINKTYPE] = (linkTable[rows, cfg.LTB_LINKTYPE] - 1000) # --------------------------------------------------------------------- # Create output geodatabase if not gp.exists(outputGDB): gp.createfilegdb(cfg.OUTPUTDIR, path.basename(outputGDB)) if cfg.useArcpy: arcpy.env.workspace = outputGDB else: gp.workspace = outputGDB gp.pyramid = "NONE" gp.rasterstatistics = "NONE" # Copy mosaic raster to output geodatabase saveFloatRaster = False if saveFloatRaster == True: floatRaster = outputGDB + '\\' + PREFIX + mosaicBaseName + '_flt' # Full path statement = 'arcObj.CopyRaster_management(mosaicRaster, floatRaster)' try: exec statement except: pass # --------------------------------------------------------------------- # convert mosaic raster to integer intRaster = path.join(outputGDB, PREFIX + mosaicBaseName) if cfg.useArcpy: statement = ('outras = Int(Raster(mosaicRaster) - offset + 0.5); ' 'outras.save(intRaster)') else: expression = "int(" + mosaicRaster + " - " + str( offset) + " + 0.5)" statement = 'gp.SingleOutputMapAlgebra_sa(expression, intRaster)' count = 0 while True: try: exec statement randomerror() except: count, tryAgain = lu.retry_arc_error(count, statement) if not tryAgain: exec statement else: break # --------------------------------------------------------------------- if writeTruncRaster: # ----------------------------------------------------------------- # Set anything beyond cfg.CWDTHRESH to NODATA. if arcpyAvailable: cfg.useArcpy = True # For Alissa Pump's error with 10.1 cutoffText = str(cfg.CWDTHRESH) if cutoffText[-6:] == '000000': cutoffText = cutoffText[0:-6] + 'm' elif cutoffText[-3:] == '000': cutoffText = cutoffText[0:-3] + 'k' truncRaster = (outputGDB + '\\' + PREFIX + mosaicBaseName + '_truncated_at_' + cutoffText) count = 0 if cfg.useArcpy: statement = ('outRas = Raster(intRaster) * ' '(Con(Raster(intRaster) <= cfg.CWDTHRESH,1)); ' 'outRas.save(truncRaster)') else: expression = ("(" + intRaster + " * (con(" + intRaster + "<= " + str(cfg.CWDTHRESH) + ",1)))") statement = ('gp.SingleOutputMapAlgebra_sa(expression, ' 'truncRaster)') count = 0 while True: try: exec statement randomerror() except: count, tryAgain = lu.retry_arc_error(count, statement) if not tryAgain: exec statement else: break cfg.useArcpy = False # End fix for Alissa Pump's error with 10.1 # --------------------------------------------------------------------- # Check for unreasonably low minimum NLCC values try: mosaicGrid = path.join(cfg.LCCBASEDIR, 'mos') # Copy to grid to test arcObj.CopyRaster_management(mosaicRaster, mosaicGrid) minObject = gp.GetRasterProperties(mosaicGrid, "MINIMUM") rasterMin = float(str(minObject.getoutput(0))) except: gp.AddWarning('\n------------------------------------------------') gp.AddWarning( 'WARNING: Raster minimum check failed in step 5. \n' 'This may mean the output rasters are corrupted. Please \n' 'be sure to check for valid rasters in ' + outputGDB) rasterMin = 0 tolerance = (float(gp.cellSize) * -10) if rasterMin < tolerance: lu.dashline(1) msg = ('WARNING: Minimum value of mosaicked corridor map is ' 'much less than zero (' + str(rasterMin) + ').' '\nThis could mean that BOUNDING CIRCLE BUFFER DISTANCES ' 'were too small and a corridor passed outside of a ' 'bounding circle, or that a corridor passed outside of the ' 'resistance map. \n') gp.AddWarning(msg) gprint('\nWriting final LCP maps...') if cfg.STEP4: finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=4, thisStep=5) elif cfg.STEP3: finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=3, thisStep=5) else: # Don't know if step 4 was run, since this is started at step 5. # Use presence of previous linktable files to figure this out. # Linktable name includes step number. prevLinkTableFile = lu.get_prev_step_link_table(step=5) prevStepInd = len(prevLinkTableFile) - 5 lastStep = prevLinkTableFile[prevStepInd] finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep, thisStep=5) outlinkTableFile = lu.get_this_step_link_table(step=5) gprint('Updating ' + outlinkTableFile) lu.write_link_table(linkTable, outlinkTableFile) linkTableLogFile = path.join(cfg.LOGDIR, "linkTable_s5.csv") lu.write_link_table(linkTable, linkTableLogFile) linkTableFinalFile = path.join(cfg.OUTPUTDIR, PREFIX + "_linkTable_s5.csv") lu.write_link_table(finalLinkTable, linkTableFinalFile) gprint('Copy of final linkTable written to ' + linkTableFinalFile) gprint('Creating shapefiles with linework for links.') try: lu.write_link_maps(outlinkTableFile, step=5) except: lu.write_link_maps(outlinkTableFile, step=5) # Create final linkmap files in output directory, and remove files from # scratch. lu.copy_final_link_maps(step=5) if not SAVENORMLCCS: lu.delete_dir(cfg.LCCBASEDIR) # Build statistics for corridor rasters gp.addmessage('\nBuilding output statistics and pyramids ' 'for corridor raster') lu.build_stats(intRaster) if writeTruncRaster: gp.addmessage('Building output statistics ' 'for truncated corridor raster') lu.build_stats(truncRaster) # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.dashline(1) gprint('****Failed in step 5. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 5. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME) return
def do_radius_loop(): """Do radius loop.""" link_table = link_table_tmp.copy() start_time = time.clock() link_loop = 0 pct_done = 0 gprint('\nMapping barriers at a radius of ' + str(radius) + ' ' + str(map_units)) if cfg.SUM_BARRIERS: gprint('using SUM method') else: gprint('using MAXIMUM method') if num_corridor_links > 1: gprint('0 percent done') last_mosaic_ras = None last_mosaic_ras_pct = None for x in range(0, num_links): pct_done = lu.report_pct_done( link_loop, num_corridor_links, pct_done) if ((link_table[x, cfg.LTB_LINKTYPE] > 0) and (link_table[x, cfg.LTB_LINKTYPE] < 1000)): link_loop = link_loop + 1 # source and target cores corex = int(core_list[x, 0]) corey = int(core_list[x, 1]) # Get cwd rasters for source and target cores cwd_ras1 = lu.get_cwd_path(corex) cwd_ras2 = lu.get_cwd_path(corey) # Mask out areas above CWD threshold cwd_tmp1 = None cwd_tmp2 = None if cfg.BARRIER_CWD_THRESH is not None: if x == 1: lu.dashline(1) gprint(' Using CWD threshold of ' + str(cfg.BARRIER_CWD_THRESH) + ' map units.') arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.env.snapRaster = cfg.RESRAST cwd_tmp1 = path.join(cfg.SCRATCHDIR, "tmp" + str(corex)) out_con = arcpy.sa.Con( cwd_ras1 < float(cfg.BARRIER_CWD_THRESH), cwd_ras1) out_con.save(cwd_tmp1) cwd_ras1 = cwd_tmp1 cwd_tmp2 = path.join(cfg.SCRATCHDIR, "tmp" + str(corey)) out_con = arcpy.sa.Con( cwd_ras2 < float(cfg.BARRIER_CWD_THRESH), cwd_ras2) out_con.save(cwd_tmp2) cwd_ras2 = cwd_tmp2 focal_ras1 = lu.get_focal_path(corex, radius) focal_ras2 = lu.get_focal_path(corey, radius) link = lu.get_links_from_core_pairs(link_table, corex, corey) lc_dist = float(link_table[link, cfg.LTB_CWDIST]) # Detect barriers at radius using neighborhood stats # Create the Neighborhood Object inner_radius = radius - 1 outer_radius = radius dia = 2 * radius in_neighborhood = ("ANNULUS " + str(inner_radius) + " " + str(outer_radius) + " MAP") @Retry(10) def exec_focal(): """Execute focal statistics.""" if not path.exists(focal_ras1): arcpy.env.extent = cwd_ras1 out_focal_stats = arcpy.sa.FocalStatistics( cwd_ras1, in_neighborhood, "MINIMUM", "DATA") if SET_CORES_TO_NULL: # Set areas overlapping cores to NoData xxx out_focal_stats2 = arcpy.sa.Con( out_focal_stats > 0, out_focal_stats) out_focal_stats2.save(focal_ras1) else: out_focal_stats.save(focal_ras1) arcpy.env.extent = cfg.RESRAST if not path.exists(focal_ras2): arcpy.env.extent = cwd_ras2 out_focal_stats = arcpy.sa.FocalStatistics( cwd_ras2, in_neighborhood, "MINIMUM", "DATA") if SET_CORES_TO_NULL: # Set areas overlapping cores to NoData xxx out_focal_stats2 = arcpy.sa.Con( out_focal_stats > 0, out_focal_stats) out_focal_stats2.save(focal_ras2) else: out_focal_stats.save(focal_ras2) arcpy.env.extent = cfg.RESRAST exec_focal() lu.delete_data(cwd_tmp1) lu.delete_data(cwd_tmp2) barrier_ras = path.join( cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey)+'.tif') # Need to set nulls to 0, # also create trim rasters as we go if cfg.SUM_BARRIERS: out_ras = ((lc_dist - arcpy.sa.Raster(focal_ras1) - arcpy.sa.Raster(focal_ras2) - dia) / dia) out_con = arcpy.sa.Con(arcpy.sa.IsNull(out_ras), 0, out_ras) out_con2 = arcpy.sa.Con(out_con < 0, 0, out_con) out_con2.save(barrier_ras) # Execute FocalStatistics to fill out search radii in_neighborhood = ("CIRCLE " + str(outer_radius) + " MAP") fill_ras = path.join( cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey) + "_fill.tif") out_focal_stats = arcpy.sa.FocalStatistics( barrier_ras, in_neighborhood, "MAXIMUM", "DATA") out_focal_stats.save(fill_ras) if cfg.WRITE_TRIM_RASTERS: trm_ras = path.join( cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey) + "_trim.tif") ras_list = [fill_ras, resist_fill_ras] out_cell_statistics = arcpy.sa.CellStatistics( ras_list, "MINIMUM") out_cell_statistics.save(trm_ras) else: @Retry(10) def clac_ben(): """Calculate potential benefit. Calculate potential benefit per map unit restored. """ out_ras = ( (lc_dist - arcpy.sa.Raster(focal_ras1) - arcpy.sa.Raster(focal_ras2) - dia) / dia) out_ras.save(barrier_ras) clac_ben() if cfg.WRITE_PCT_RASTERS: # Calculate % potential benefit per unit restored barrier_ras_pct = path.join( cbarrierdir, "b" + str(radius) + "_" + str(corex) + "_" + str(corey) + '_pct.tif') @Retry(10) def calc_ben_pct(): """Calc benefit percentage.""" outras = (100 * (arcpy.sa.Raster(barrier_ras) / lc_dist)) outras.save(barrier_ras_pct) calc_ben_pct() # Mosaic barrier results across core area pairs mosaic_dir = path.join(cfg.SCRATCHDIR, 'mos' + str(rad_id) + '_' + str(x + 1)) lu.create_dir(mosaic_dir) mos_fn = 'mos_temp' tmp_mosaic_ras = path.join(mosaic_dir, mos_fn) tmp_mosaic_ras_trim = path.join(mosaic_dir, 'mos_temp_trm') arcpy.env.workspace = mosaic_dir if link_loop == 1: last_mosaic_ras_trim = None # For first grid copy rather than mosaic arcpy.CopyRaster_management(barrier_ras, tmp_mosaic_ras) if cfg.SUM_BARRIERS and cfg.WRITE_TRIM_RASTERS: arcpy.CopyRaster_management( trm_ras, tmp_mosaic_ras_trim) else: if cfg.SUM_BARRIERS: out_con = arcpy.sa.Con( arcpy.sa.Raster(barrier_ras) < 0, last_mosaic_ras, arcpy.sa.Raster(barrier_ras) + arcpy.sa.Raster(last_mosaic_ras)) out_con.save(tmp_mosaic_ras) if cfg.WRITE_TRIM_RASTERS: out_con = arcpy.sa.Con( arcpy.sa.Raster(trm_ras) < 0, last_mosaic_ras_trim, arcpy.sa.Raster(trm_ras) + arcpy.sa.Raster(last_mosaic_ras_trim) ) out_con.save(tmp_mosaic_ras_trim) else: in_rasters = (";".join([barrier_ras, last_mosaic_ras])) @Retry(10) def mosaic_to_new(): """Mosaic to new raster.""" arcpy.MosaicToNewRaster_management( input_rasters=in_rasters, output_location=mosaic_dir, raster_dataset_name_with_extension\ =mos_fn, pixel_type="32_BIT_FLOAT", cellsize=arcpy.env.cellSize, number_of_bands="1", mosaic_method="MAXIMUM") mosaic_to_new() if link_loop > 1: # Clean up from previous loop lu.delete_data(last_mosaic_ras) last_mosaic_dir = path.dirname(last_mosaic_ras) lu.clean_out_workspace(last_mosaic_dir) lu.delete_dir(last_mosaic_dir) last_mosaic_ras = tmp_mosaic_ras if cfg.WRITE_TRIM_RASTERS: last_mosaic_ras_trim = tmp_mosaic_ras_trim if cfg.WRITE_PCT_RASTERS: mos_pct_fn = 'mos_temp_pct' mosaic_dir_pct = path.join(cfg.SCRATCHDIR, 'mosP' + str(rad_id) + '_' + str(x+1)) lu.create_dir(mosaic_dir_pct) tmp_mosaic_ras_pct = path.join(mosaic_dir_pct, mos_pct_fn) if link_loop == 1: # If this is the first grid then copy # rather than mosaic if cfg.SUM_BARRIERS: out_con = arcpy.sa.Con( arcpy.sa.Raster(barrier_ras_pct) < 0, 0, arcpy.sa.Con(arcpy.sa.IsNull (barrier_ras_pct), 0, barrier_ras_pct)) out_con.save(tmp_mosaic_ras_pct) else: arcpy.CopyRaster_management( barrier_ras_pct, tmp_mosaic_ras_pct) else: if cfg.SUM_BARRIERS: @Retry(10) def sum_barriers(): """Sum barriers.""" out_con = arcpy.sa.Con( arcpy.sa.Raster(barrier_ras_pct) < 0, last_mosaic_ras_pct, arcpy.sa.Raster(barrier_ras_pct) + arcpy.sa.Raster( last_mosaic_ras_pct)) out_con.save(tmp_mosaic_ras_pct) sum_barriers() else: in_rasters = (";".join([barrier_ras_pct, last_mosaic_ras_pct])) @Retry(10) def max_barriers(): """Get max barriers.""" arcpy.MosaicToNewRaster_management( input_rasters=in_rasters, output_location=mosaic_dir_pct, raster_dataset_name_with_extension =mos_pct_fn, pixel_type="32_BIT_FLOAT", cellsize=arcpy.env.cellSize, number_of_bands="1", mosaic_method="MAXIMUM") max_barriers() if link_loop > 1: # Clean up from previous loop lu.delete_data(last_mosaic_ras_pct) last_mosaic_dir_pct = path.dirname( last_mosaic_ras_pct) lu.clean_out_workspace(last_mosaic_dir_pct) lu.delete_dir(last_mosaic_dir_pct) last_mosaic_ras_pct = tmp_mosaic_ras_pct if not cfg.SAVEBARRIERRASTERS: lu.delete_data(barrier_ras) if cfg.WRITE_PCT_RASTERS: lu.delete_data(barrier_ras_pct) if cfg.WRITE_TRIM_RASTERS: lu.delete_data(trm_ras) # Temporarily disable links in linktable - # don't want to mosaic them twice for y in range(x + 1, num_links): corex1 = int(core_list[y, 0]) corey1 = int(core_list[y, 1]) if corex1 == corex and corey1 == corey: link_table[y, cfg.LTB_LINKTYPE] = ( link_table[y, cfg.LTB_LINKTYPE] + 1000) elif corex1 == corey and corey1 == corex: link_table[y, cfg.LTB_LINKTYPE] = ( link_table[y, cfg.LTB_LINKTYPE] + 1000) if num_corridor_links > 1 and pct_done < 100: gprint('100 percent done') gprint('Summarizing barrier data for search radius.') # Rows that were temporarily disabled rows = npy.where(link_table[:, cfg.LTB_LINKTYPE] > 1000) link_table[rows, cfg.LTB_LINKTYPE] = ( link_table[rows, cfg.LTB_LINKTYPE] - 1000) # ----------------------------------------------------------------- # Set negative values to null or zero and write geodatabase. mosaic_fn = (prefix + "_BarrierCenters" + sum_suffix + "_Rad" + str(radius)) mosaic_ras = path.join(cfg.BARRIERGDB, mosaic_fn) arcpy.env.extent = cfg.RESRAST out_set_null = arcpy.sa.SetNull(tmp_mosaic_ras, tmp_mosaic_ras, "VALUE < 0") # xxx orig out_set_null.save(mosaic_ras) lu.delete_data(tmp_mosaic_ras) if cfg.SUM_BARRIERS and cfg.WRITE_TRIM_RASTERS: mosaic_fn = (prefix + "_BarrierCircles_RBMin" + sum_suffix + "_Rad" + str(radius)) mosaic_ras_trim = path.join(cfg.BARRIERGDB, mosaic_fn) arcpy.CopyRaster_management(tmp_mosaic_ras_trim, mosaic_ras_trim) lu.delete_data(tmp_mosaic_ras) if cfg.WRITE_PCT_RASTERS: # Do same for percent raster mosaic_pct_fn = (prefix + "_BarrierCenters_Pct" + sum_suffix + "_Rad" + str(radius)) arcpy.env.extent = cfg.RESRAST out_set_null = arcpy.sa.SetNull(tmp_mosaic_ras_pct, tmp_mosaic_ras_pct, "VALUE < 0") mosaic_ras_pct = path.join(cfg.BARRIERGDB, mosaic_pct_fn) out_set_null.save(mosaic_ras_pct) lu.delete_data(tmp_mosaic_ras_pct) # 'Grow out' maximum restoration gain to # neighborhood size for display in_neighborhood = "CIRCLE " + str(outer_radius) + " MAP" # Execute FocalStatistics fill_ras_fn = "barriers_fill" + str(outer_radius) + TIF fill_ras = path.join(cfg.BARRIERBASEDIR, fill_ras_fn) out_focal_stats = arcpy.sa.FocalStatistics( mosaic_ras, in_neighborhood, "MAXIMUM", "DATA") out_focal_stats.save(fill_ras) if cfg.WRITE_PCT_RASTERS: # Do same for percent raster fill_ras_pct_fn = ( "barriers_fill_pct" + str(outer_radius) + TIF) fill_ras_pct = path.join(cfg.BARRIERBASEDIR, fill_ras_pct_fn) out_focal_stats = arcpy.sa.FocalStatistics( mosaic_ras_pct, in_neighborhood, "MAXIMUM", "DATA") out_focal_stats.save(fill_ras_pct) # Place copies of filled rasters in output geodatabase arcpy.env.workspace = cfg.BARRIERGDB fill_ras_fn = (prefix + "_BarrrierCircles" + sum_suffix + "_Rad" + str(outer_radius)) arcpy.CopyRaster_management(fill_ras, fill_ras_fn) if cfg.WRITE_PCT_RASTERS: fill_ras_pct_fn = (prefix + "_BarrrierCircles_Pct" + sum_suffix + "_Rad" + str(outer_radius)) arcpy.CopyRaster_management(fill_ras_pct, fill_ras_pct_fn) if not cfg.SUM_BARRIERS and cfg.WRITE_TRIM_RASTERS: # Create pared-down version of filled raster- remove pixels # that don't need restoring by allowing a pixel to only # contribute its resistance value to restoration gain out_ras_fn = "barriers_trm" + str(outer_radius) + TIF out_ras = path.join(cfg.BARRIERBASEDIR, out_ras_fn) ras_list = [fill_ras, resist_fill_ras] out_cell_statistics = arcpy.sa.CellStatistics(ras_list, "MINIMUM") out_cell_statistics.save(out_ras) # SECOND ROUND TO CLIP BY DATA VALUES IN BARRIER RASTER out_ras_2fn = ("barriers_trm" + sum_suffix + str(outer_radius) + "_2" + TIF) out_ras2 = path.join(cfg.BARRIERBASEDIR, out_ras_2fn) output = arcpy.sa.Con(arcpy.sa.IsNull(fill_ras), fill_ras, out_ras) output.save(out_ras2) out_ras_fn = (prefix + "_BarrierCircles_RBMin" + sum_suffix + "_Rad" + str(outer_radius)) arcpy.CopyRaster_management(out_ras2, out_ras_fn) start_time = lu.elapsed_time(start_time)
def gen_cwd_back(core_list, climate_lyr, resist_lyr, core_lyr): """"Generate CWD and back rasters using r.walk in GRASS""" slope_factor = "1" walk_coeff_flat = "1" walk_coeff_uphill = str(cc_env.climate_cost) walk_coeff_downhill = str(cc_env.climate_cost * -1) walk_coeff = (walk_coeff_flat + "," + walk_coeff_uphill + "," + walk_coeff_downhill + "," + walk_coeff_downhill) focal_core_rast = "focal_core_rast" gcwd = "gcwd" gback = "gback" gbackrc = "gbackrc" core_points = "corepoints" no_cores = str(len(core_list)) # Map from directional degree output from GRASS to Arc's 1 to 8 directions # format. See r.walk source code and ArcGIS's 'Understanding cost distance # analysis' help page. rc_rules = "180=5\n225=4\n270=3\n315=2\n360=1\n45=8\n90=7\n135=6" try: for position, core_no in enumerate(core_list): core_no_txt = str(core_no) lm_util.gprint("Generating CWD and back rasters for Core " + core_no_txt + " (" + str(position + 1) + "/" + no_cores + ")") # Pull out focal core for cwd analysis write_grass_cmd("r.reclass", input=core_lyr, output=focal_core_rast, overwrite=True, rules="-", stdin=core_no_txt + '=' + core_no_txt) # Converting raster core to point feature run_grass_cmd("r.to.vect", flags="z", input=focal_core_rast, output=core_points, type="point") # Running r.walk to create CWD and back raster run_grass_cmd("r.walk", elevation=climate_lyr, friction=resist_lyr, output=gcwd, outdir=gback, start_points=core_points, walk_coeff=walk_coeff, slope_factor=slope_factor) # Reclassify back raster directional degree output to ArcGIS format write_grass_cmd("r.reclass", input=gback, output=gbackrc, rules="-", stdin=rc_rules) # Get spatial reference for defining ARCINFO raster projections desc_data = arcpy.Describe(cc_env.prj_core_rast) spatial_ref = desc_data.spatialReference # Get cwd path (e.g. ..\datapass\cwd\cw\cwd_3) cwd_path = lm_util.get_cwd_path(core_no) def create_arcgrid(rtype, grass_grid): """Export GRASS raster to ASCII grid and then to ARCINFO grid """ ascii_grid = os.path.join(cc_env.out_dir, rtype + core_no_txt + ".asc") arc_grid = cwd_path.replace("cwd_", rtype) run_grass_cmd("r.out.arc", input=grass_grid, output=ascii_grid) arcpy.CopyRaster_management(ascii_grid, arc_grid) arcpy.DefineProjection_management(arc_grid, spatial_ref) os.remove(ascii_grid) create_arcgrid("cwd_", gcwd) # Export CWD raster create_arcgrid("back_", gbackrc) # Export reclassified back raster except Exception: raise