def STEP3_calc_cwds(): """Calculates cost-weighted distances from each core area. Uses bounding circles around source and target cores to limit extent of cwd calculations and speed computation. """ try: lu.dashline(1) gprint('Running script ' + _SCRIPT_NAME) lu.dashline(0) # Super secret setting to re-start failed run. Enter 'RESTART' as the # Name of the pairwise distance table in step 2, and uncheck step 2. # We can eventually place this in a .ini file. rerun = False if cfg.S2EUCDISTFILE != None: if cfg.S2EUCDISTFILE.lower() == "restart": rerun = True # if cfg.TMAXCWDIST is None: # gprint('NOT using a maximum cost-weighted distance.') # else: # gprint('Max cost-weighted distance for CWD calcs set ' # 'to ' + str(cfg.TMAXCWDIST) + '\n') if (cfg.BUFFERDIST) is not None: gprint('Bounding circles plus a buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units will ' 'be used \n to limit extent of cost distance ' 'calculations.') elif cfg.TOOL <> cfg.TOOL_CC: gprint('NOT using bounding circles in cost distance ' 'calculations.') # set the analysis extent and cell size # So we don't extract rasters that go beyond extent of original raster if arcpy: arcpy.env.cellSize = cfg.RESRAST arcpy.env.extent="MINOF" else: gp.cellSize = gp.Describe(cfg.RESRAST).MeanCellHeight gp.Extent = "MINOF" gp.mask = cfg.RESRAST if arcpy: arcpy.env.overwriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR else: gp.OverwriteOutput = True gp.workspace = cfg.SCRATCHDIR gp.scratchWorkspace = cfg.ARCSCRATCHDIR # Load linkTable (created in previous script) linkTableFile = lu.get_prev_step_link_table(step=3) linkTable = lu.load_link_table(linkTableFile) lu.report_links(linkTable) # Identify cores to map from LinkTable coresToMap = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) numCoresToMap = len(coresToMap) if numCoresToMap < 3: # No need to check for intermediate cores, because there aren't any cfg.S3DROPLCCSic = False else: cfg.S3DROPLCCSic = cfg.S3DROPLCCS gprint('\nNumber of core areas to connect: ' + str(numCoresToMap)) if rerun: # If picking up a failed run, make sure needed files are there lu.dashline(1) gprint ('\n****** RESTART MODE ENABLED ******\n') gprint ('**** NOTE: This mode picks up step 3 where a\n' 'previous run left off due to a crash or user\n' 'abort. It assumes you are using the same input\n' 'data used in the terminated run.\n\n') lu.warn('IMPORTANT: Your LCP and stick feature classes\n' 'will LOSE LCPs that were already created, but\n' 'your final raster corridor map should be complete.\n') lu.dashline(0) lu.snooze(10) savedLinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") coreListFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") if not path.exists(savedLinkTableFile) or not path.exists( coreListFile): gprint('No partial results file found from previous ' 'stopped run. Starting run from beginning.\n') lu.dashline(0) rerun = False # If picking up a failed run, use old folders if not rerun: startIndex = 0 if cfg.TOOL <> cfg.TOOL_CC: lu.make_cwd_paths(max(coresToMap)) # Set up cwd directories # make a feature layer for input cores to select from gp.MakeFeatureLayer(cfg.COREFC, cfg.FCORES) # Drop links that are too long gprint('\nChecking for corridors that are too long to map.') DISABLE_LEAST_COST_NO_VAL = False linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, 0, cfg.MAXCOSTDIST, 0, DISABLE_LEAST_COST_NO_VAL) # ------------------------------------------------------------------ # Bounding boxes if (cfg.BUFFERDIST) is not None: # create bounding boxes around cores start_time = time.clock() # lu.dashline(1) gprint('Calculating bounding boxes for core areas.') extentBoxList = npy.zeros((0,5), dtype='float32') for x in range(len(coresToMap)): core = coresToMap[x] boxCoords = lu.get_extent_box_coords(core) extentBoxList = npy.append(extentBoxList, boxCoords, axis=0) gprint('\nDone calculating bounding boxes.') start_time = lu.elapsed_time(start_time) # lu.dashline() # Bounding circle code if cfg.BUFFERDIST is not None: # Make a set of circles encompassing core areas we'll be connecting start_time = time.clock() gprint('Calculating bounding circles around potential' ' corridors.') # x y corex corey radius- stores data for bounding circle centroids boundingCirclePointArray = npy.zeros((0,5), dtype='float32') circleList = npy.zeros((0,3), dtype='int32') numLinks = linkTable.shape[0] for x in range(0, numLinks): if ((linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_CORR) or (linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_KEEP)): # if it's a valid corridor link linkId = int(linkTable[x,cfg.LTB_LINKID]) # fixme- this code is clumsy- can trim down cores = npy.zeros((1,3), dtype='int32') cores[0,:] = npy.sort([0, linkTable[x,cfg.LTB_CORE1], linkTable[x,cfg.LTB_CORE2]]) corex = cores[0,1] corey = cores[0,2] cores[0,0] = linkId ################### foundFlag = False for y in range(0,len(circleList)): # clumsy if (circleList[y,1] == corex and circleList[y,2] == corey): foundFlag = True if not foundFlag: circlePointData = ( lu.get_bounding_circle_data(extentBoxList, corex, corey, cfg.BUFFERDIST)) boundingCirclePointArray = ( npy.append(boundingCirclePointArray, circlePointData, axis=0)) # keep track of which cores we draw bounding circles # around circleList = npy.append(circleList, cores, axis=0) gprint('\nCreating bounding circles using buffer ' 'analysis.') dir, BNDCIRCENS = path.split(cfg.BNDCIRCENS) lu.make_points(cfg.SCRATCHDIR, boundingCirclePointArray, BNDCIRCENS) lu.delete_data(cfg.BNDCIRS) gp.buffer_analysis(cfg.BNDCIRCENS, cfg.BNDCIRS, "radius") gp.deletefield (cfg.BNDCIRS, "BUFF_DIST") gprint('Successfully created bounding circles around ' 'potential corridors using \na buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.') start_time = lu.elapsed_time(start_time) gprint('Reducing global processing area using bounding ' 'circle plus buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.\n') extentBoxList = npy.zeros((0,5),dtype='float32') boxCoords = lu.get_extent_box_coords() extentBoxList = npy.append(extentBoxList,boxCoords,axis=0) extentBoxList[0,0] = 0 boundingCirclePointArray = npy.zeros((0,5),dtype='float32') circlePointData=lu.get_bounding_circle_data(extentBoxList, 0, 0, cfg.BUFFERDIST) dir, BNDCIRCEN = path.split(cfg.BNDCIRCEN) lu.make_points(cfg.SCRATCHDIR, circlePointData, BNDCIRCEN) lu.delete_data(cfg.BNDCIR) gp.buffer_analysis(cfg.BNDCIRCEN, cfg.BNDCIR, "radius") gprint('Extracting raster....') cfg.BOUNDRESIS = cfg.BOUNDRESIS + tif lu.delete_data(cfg.BOUNDRESIS) count = 0 statement = ( 'gp.ExtractByMask_sa(cfg.RESRAST, cfg.BNDCIR, cfg.BOUNDRESIS)') while True: try: exec statement randomerror() except: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec statement else: break gprint('\nReduced resistance raster extracted using ' 'bounding circle.') else: #if not using bounding circles, just go with resistance raster. cfg.BOUNDRESIS = cfg.RESRAST # --------------------------------------------------------------------- # Rasterize core areas to speed cost distance calcs # lu.dashline(1) gprint("Creating core area raster.") gp.SelectLayerByAttribute(cfg.FCORES, "CLEAR_SELECTION") if arcpy: arcpy.env.cellSize = cfg.BOUNDRESIS arcpy.env.extent = cfg.BOUNDRESIS else: gp.cellSize = gp.Describe(cfg.BOUNDRESIS).MeanCellHeight gp.extent = gp.Describe(cfg.BOUNDRESIS).extent if rerun: # saved linktable replaces the one now in memory linkTable = lu.load_link_table(savedLinkTableFile) coresToMapSaved = npy.loadtxt(coreListFile, dtype='Float64', comments='#', delimiter=',') startIndex = coresToMapSaved[0] # Index of core where we left off del coresToMapSaved gprint ('\n****** Re-starting run at core area number ' + str(int(coresToMap[startIndex]))+ ' ******\n') lu.dashline(0) if arcpy: arcpy.env.extent = "MINOF" else: gp.extent = "MINOF" #---------------------------------------------------------------------- # Loop through cores, do cwd calcs for each if cfg.TOOL == cfg.TOOL_CC: gprint("\nMapping least-cost paths.\n") else: gprint("\nStarting cost distance calculations.\n") lcpLoop = 0 failures = 0 x = startIndex endIndex = len(coresToMap) linkTableMod = linkTable.copy() while x < endIndex: startTime1 = time.clock() # Modification of linkTable in function was causing problems. so # make a copy: linkTablePassed = linkTableMod.copy() (linkTableReturned, failures, lcpLoop) = do_cwd_calcs(x, linkTablePassed, coresToMap, lcpLoop, failures) if failures == 0: # If iteration was successful, continue with next core linkTableMod = linkTableReturned sourceCore = int(coresToMap[x]) gprint('Done with all calculations for core ID #' + str(sourceCore) + '. ' + str(int(x + 1)) + ' of ' + str(endIndex) + ' cores have been processed.') start_time = lu.elapsed_time(startTime1) outlinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.write_link_table(linkTableMod, outlinkTableFile) # Increment loop counter x = x + 1 else: # If iteration failed, try again after a wait period delay_restart(failures) #---------------------------------------------------------------------- linkTable = linkTableMod # reinstate temporarily disabled links rows = npy.where(linkTable[:,cfg.LTB_LINKTYPE] > 1000) linkTable[rows,cfg.LTB_LINKTYPE] = (linkTable[rows,cfg.LTB_LINKTYPE] - 1000) # Drop links that are too long DISABLE_LEAST_COST_NO_VAL = True linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, cfg.MINEUCDIST, cfg.MAXCOSTDIST, cfg.MINCOSTDIST, DISABLE_LEAST_COST_NO_VAL) # Write link table file outlinkTableFile = lu.get_this_step_link_table(step=3) gprint('Updating ' + outlinkTableFile) lu.write_link_table(linkTable, outlinkTableFile) linkTableLogFile = path.join(cfg.LOGDIR, "linkTable_s3.csv") lu.write_link_table(linkTable, linkTableLogFile) start_time = time.clock() gprint('Creating shapefiles with linework for links...') try: lu.write_link_maps(outlinkTableFile, step=3) except: lu.write_link_maps(outlinkTableFile, step=3) start_time = lu.elapsed_time(start_time) gprint('\nIndividual cost-weighted distance layers written ' 'to "cwd" directory. \n') gprint(outlinkTableFile + '\n updated with cost-weighted distances between core areas.') #Clean up temporary files for restart code tempFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") lu.delete_file(tempFile) tempFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.delete_file(tempFile) # Check if climate tool is calling linkage mapper if cfg.TOOL == cfg.TOOL_CC: coreList = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) for core in coreList: cwdRaster = lu.get_cwd_path(int(core)) back_rast = cwdRaster.replace("cwd_", "back_") lu.delete_data(back_rast) # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME) return
def STEP8_calc_pinchpoints(): """ Maps pinch points in Linkage Mapper corridors using Circuitscape given CWD calculations from s3_calcCwds.py. """ try: lu.dashline(0) gprint('Running script ' + _SCRIPT_NAME) restartFlag = False if cfg.CWDCUTOFF < 0: cfg.CWDCUTOFF = cfg.CWDCUTOFF * -1 restartFlag = True # Restart code in progress CSPATH = lu.get_cs_path() outputGDB = path.join(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) arcpy.OverWriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.pyramid = "NONE" arcpy.env.rasterstatistics = "NONE" # set the analysis extent and cell size to that of the resistance # surface arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.snapraster = cfg.RESRAST resRaster = cfg.RESRAST arcpy.env.extent = "MINOF" minObject = arcpy.GetRasterProperties_management(resRaster, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) if rasterMin <= 0: msg = ( 'Error: resistance raster cannot have 0 or negative values.') lu.raise_error(msg) if cfg.DO_ADJACENTPAIRS: prevLcpShapefile = lu.get_lcp_shapefile(None, thisStep=8) if not arcpy.Exists(prevLcpShapefile): msg = ('Cannot find an LCP shapefile from step 5. Please ' 'rerun that step and any previous ones if necessary.') lu.raise_error(msg) # Remove lcp shapefile lcpShapefile = path.join(cfg.DATAPASSDIR, "lcpLines_s8.shp") lu.delete_data(lcpShapefile) inLinkTableFile = lu.get_prev_step_link_table(step=8) linkTable = lu.load_link_table(inLinkTableFile) numLinks = linkTable.shape[0] numCorridorLinks = lu.report_links(linkTable) if numCorridorLinks == 0: lu.dashline(1) msg = ('\nThere are no linkages. Bailing.') lu.raise_error(msg) if linkTable.shape[1] < 16: # If linktable has no entries from prior # centrality or pinchpint analyses extraCols = npy.zeros((numLinks, 6), dtype="float64") linkTable = linkTable[:, 0:10] linkTable = npy.append(linkTable, extraCols, axis=1) linkTable[:, cfg.LTB_LCPLEN] = -1 linkTable[:, cfg.LTB_CWDEUCR] = -1 linkTable[:, cfg.LTB_CWDPATHR] = -1 linkTable[:, cfg.LTB_EFFRESIST] = -1 linkTable[:, cfg.LTB_CWDTORR] = -1 linkTable[:, cfg.LTB_CURRENT] = -1 del extraCols # set up directories for circuit and circuit mosaic grids # Create output geodatabase if not arcpy.Exists(cfg.PINCHGDB): arcpy.CreateFileGDB_management(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) mosaicRaster = path.join(cfg.CIRCUITBASEDIR, "current_mos" + tif) coresToProcess = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) maxCoreNum = max(coresToProcess) del coresToProcess lu.dashline(0) coreList = linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1] coreList = npy.sort(coreList) #gprint('There are ' + str(len(npy.unique(coreList))) ' core areas.') INCIRCUITDIR = cfg.CIRCUITBASEDIR OUTCIRCUITDIR = path.join(cfg.CIRCUITBASEDIR, cfg.CIRCUITOUTPUTDIR_NM) CONFIGDIR = path.join(INCIRCUITDIR, cfg.CIRCUITCONFIGDIR_NM) # Cutoff value text to append to filenames cutoffText = str(cfg.CWDCUTOFF) if cutoffText[-6:] == '000000': cutoffText = cutoffText[0:-6] + 'm' elif cutoffText[-3:] == '000': cutoffText = cutoffText[0:-3] + 'k' if cfg.SQUARERESISTANCES: # Square resistance values squaredRaster = path.join(cfg.SCRATCHDIR, 'res_sqr') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR outRas = Raster(resRaster) * Raster(resRaster) outRas.save(squaredRaster) resRaster = squaredRaster if cfg.DO_ADJACENTPAIRS: linkLoop = 0 lu.dashline(1) gprint('Mapping pinch points in individual corridors \n' 'using Circuitscape.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(2) for x in range(0, numLinks): linkId = str(int(linkTable[x, cfg.LTB_LINKID])) if not (linkTable[x, cfg.LTB_LINKTYPE] > 0): continue linkLoop = linkLoop + 1 linkDir = path.join(cfg.SCRATCHDIR, 'link' + linkId) if restartFlag == True and path.exists(linkDir): gprint('continuing') continue restartFlag = False lu.create_dir(linkDir) start_time1 = time.clock() # source and target cores corex = int(coreList[x, 0]) corey = int(coreList[x, 1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) lccNormRaster = path.join(linkDir, 'lcc_norm') arcpy.env.extent = "MINOF" link = lu.get_links_from_core_pairs(linkTable, corex, corey) lcDist = float(linkTable[link, cfg.LTB_CWDIST]) # Normalized lcc rasters are created by adding cwd rasters # and subtracting the least cost distance between them. outRas = Raster(cwdRaster1) + Raster(cwdRaster2) - lcDist outRas.save(lccNormRaster) #create raster mask resMaskRaster = path.join(linkDir, 'res_mask' + tif) #create raster mask outCon = arcpy.sa.Con( Raster(lccNormRaster) <= cfg.CWDCUTOFF, 1) outCon.save(resMaskRaster) # Convert to poly. Use as mask to clip resistance raster. resMaskPoly = path.join(linkDir, 'res_mask_poly.shp') arcpy.RasterToPolygon_conversion(resMaskRaster, resMaskPoly, "NO_SIMPLIFY") arcpy.env.extent = resMaskPoly # Includes 0 values in some cases with CP LI model if tif # so using ESRI Grid format resClipRasterMasked = path.join(linkDir, 'res_clip_m') # Extract masked resistance raster. # Needs to be float to get export to npy to work. outRas = arcpy.sa.ExtractByMask(resRaster, resMaskPoly) + 0.0 outRas.save(resClipRasterMasked) resNpyFN = 'resistances_link_' + linkId + '.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy( resClipRasterMasked, resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) lu.warn('Warning:') lu.warn('Circuitscape can only solve 2-3 million nodes') lu.warn( 'per gigabyte of available RAM. \nTotal physical RAM' ' on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~' + str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(2) corePairRaster = path.join(linkDir, 'core_pairs' + tif) arcpy.env.extent = resClipRasterMasked # Next result needs to be floating pt for numpy export outCon = arcpy.sa.Con( Raster(cwdRaster1) == 0, corex, arcpy.sa.Con(Raster(cwdRaster2) == 0, corey + 0.0)) outCon.save(corePairRaster) coreNpyFN = 'cores_link_' + linkId + '.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy( corePairRaster, coreNpyFile) arcpy.env.extent = "MINOF" # Set circuitscape options and call options = lu.setCircuitscapeOptions() if cfg.WRITE_VOLT_MAPS == True: options['write_volt_maps'] = True options['habitat_file'] = resNpyFile # if int(linkId) > 2: # options['habitat_file'] = 'c:\\test.dummy' options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero'] = True outputFN = 'Circuitscape_link' + linkId + '.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) if numElements > 250000: options['print_timings'] = True configFN = 'pinchpoint_config' + linkId + '.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('Processing link ID #' + str(linkId) + '. Resistance map' ' has ' + str(int(numResistanceNodes)) + ' nodes.') memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): print_failure(numResistanceNodes, memFlag, 10) numElements, numNodes = export_ras_to_npy( resClipRasterMasked, resNpyFile) memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): msg = ( '\nCircuitscape failed. See error information above.') arcpy.AddError(msg) lu.write_log(msg) exit(1) # Either set core areas to nodata in current map or # divide each by its radius currentRaster = path.join(linkDir, "current" + tif) import_npy_to_ras(currentMap, corePairRaster, currentRaster) if cfg.WRITE_VOLT_MAPS == True: voltFN = ('Circuitscape_link' + linkId + '_voltmap_' + str(corex) + '_' + str(corey) + '.npy') voltMap = path.join(OUTCIRCUITDIR, voltFN) voltRaster = path.join( outputGDB, cfg.PREFIX + "_voltMap_" + str(corex) + '_' + str(corey)) import_npy_to_ras(voltMap, corePairRaster, voltRaster) gprint('Building output statistics and pyramids ' 'for voltage raster\n') lu.build_stats(voltRaster) arcpy.env.extent = currentRaster if SETCORESTONULL: # Set core areas to NoData in current map for color ramping currentRaster2 = currentRaster + '2' + tif outCon = arcpy.sa.Con( arcpy.sa.IsNull(Raster(corePairRaster)), Raster(currentRaster)) outCon.save(currentRaster2) currentRaster = currentRaster2 arcpy.env.extent = "MAXOF" if linkLoop == 1: lu.delete_data(mosaicRaster) @retry(10) def copyRas2(): arcpy.CopyRaster_management(currentRaster, mosaicRaster) copyRas2() else: @retry(10) def mosaicRas(): arcpy.Mosaic_management(currentRaster, mosaicRaster, "MAXIMUM", "MATCH") mosaicRas() resistancesFN = ('Circuitscape_link' + linkId + '_resistances_3columns.out') resistancesFile = path.join(OUTCIRCUITDIR, resistancesFN) resistances = npy.loadtxt(resistancesFile, dtype='Float64', comments='#') resistance = float(str(arcpy.env.cellSize)) * resistances[2] linkTable[link, cfg.LTB_EFFRESIST] = resistance # Ratio if not cfg.SQUARERESISTANCES: linkTable[link, cfg.LTB_CWDTORR] = ( linkTable[link, cfg.LTB_CWDIST] / linkTable[link, cfg.LTB_EFFRESIST]) # Clean up if cfg.SAVE_TEMP_CIRCUIT_FILES == False: lu.delete_file(coreNpyFile) coreNpyBase, extension = path.splitext(coreNpyFile) lu.delete_data(coreNpyBase + '.hdr') lu.delete_file(resNpyFile) resNpyBase, extension = path.splitext(resNpyFile) lu.delete_data(resNpyBase + '.hdr') lu.delete_file(currentMap) curMapBase, extension = path.splitext(currentMap) lu.delete_data(curMapBase + '.hdr') lu.delete_data(currentRaster) lu.clean_out_workspace(linkDir) lu.delete_dir(linkDir) gprint('Finished with link ID #' + str(linkId) + '. ' + str(linkLoop) + ' out of ' + str(numCorridorLinks) + ' links have been processed.') start_time1 = lu.elapsed_time(start_time1) outputRaster = path.join( outputGDB, cfg.PREFIX + "_current_adjacentPairs_" + cutoffText) lu.delete_data(outputRaster) @retry(10) def copyRas(): arcpy.CopyRaster_management(mosaicRaster, outputRaster) copyRas() gprint('Building output statistics and pyramids ' 'for corridor pinch point raster\n') lu.build_stats(outputRaster) finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=5, thisStep=8) linkTableFile = path.join(cfg.DATAPASSDIR, "linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFile, inLinkTableFile) linkTableFinalFile = path.join( cfg.OUTPUTDIR, cfg.PREFIX + "_linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFinalFile, inLinkTableFile) gprint('Copy of linkTable written to ' + linkTableFinalFile) #fixme: update sticks? gprint('Creating shapefiles with linework for links.') lu.write_link_maps(linkTableFinalFile, step=8) # Copy final link maps to gdb. lu.copy_final_link_maps(step=8) lu.delete_data(mosaicRaster) if not cfg.DO_ALLPAIRS: # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) return lu.dashline(1) gprint('Mapping global pinch points among all\n' 'core area pairs using Circuitscape.') if cfg.ALL_PAIR_SCENARIO == 'pairwise': gprint('Circuitscape will be run in PAIRWISE mode.') else: gprint('Circuitscape will be run in ALL-TO-ONE mode.') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST S8CORE_RAS = "s8core_ras" s8CoreRasPath = path.join(cfg.SCRATCHDIR, S8CORE_RAS) arcpy.FeatureToRaster_conversion(cfg.COREFC, cfg.COREFN, s8CoreRasPath, arcpy.env.cellSize) binaryCoreRaster = path.join(cfg.SCRATCHDIR, "core_ras_bin") # The following commands cause file lock problems on save. using gp # instead. # outCon = arcpy.sa.Con(S8CORE_RAS, 1, "#", "VALUE > 0") # outCon.save(binaryCoreRaster) # gp.Con_sa(s8CoreRasPath, 1, binaryCoreRaster, "#", "VALUE > 0") outCon = arcpy.sa.Con(Raster(s8CoreRasPath) > 0, 1) outCon.save(binaryCoreRaster) s5corridorRas = path.join(cfg.OUTPUTGDB, cfg.PREFIX + "_corridors") if not arcpy.Exists(s5corridorRas): s5corridorRas = path.join(cfg.OUTPUTGDB, cfg.PREFIX + "_lcc_mosaic_int") outCon = arcpy.sa.Con( Raster(s5corridorRas) <= cfg.CWDCUTOFF, Raster(resRaster), arcpy.sa.Con(Raster(binaryCoreRaster) > 0, Raster(resRaster))) resRasClipPath = path.join(cfg.SCRATCHDIR, 'res_ras_clip') outCon.save(resRasClipPath) arcpy.env.cellSize = resRasClipPath arcpy.env.extent = resRasClipPath s8CoreRasClipped = s8CoreRasPath + '_c' # Produce core raster with same extent as clipped resistance raster # added to ensure correct data type- nodata values were positive for # cores otherwise outCon = arcpy.sa.Con(arcpy.sa.IsNull(Raster(s8CoreRasPath)), -9999, Raster(s8CoreRasPath)) outCon.save(s8CoreRasClipped) resNpyFN = 'resistances.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy( resRasClipPath, resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) lu.warn('Warning:') lu.warn('Circuitscape can only solve 2-3 million nodes') lu.warn('per gigabyte of available RAM. \nTotal physical RAM ' 'on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~' + str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(0) coreNpyFN = 'cores.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy(s8CoreRasClipped, coreNpyFile) arcpy.env.extent = "MINOF" options = lu.setCircuitscapeOptions() options['scenario'] = cfg.ALL_PAIR_SCENARIO options['habitat_file'] = resNpyFile options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero'] = True outputFN = 'Circuitscape.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) options['print_timings'] = True configFN = 'pinchpoint_allpair_config.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('\nResistance map has ' + str(int(numResistanceNodes)) + ' nodes.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(0) call_circuitscape(CSPATH, outConfigFile) # test = subprocess.call([CSPATH, outConfigFile], # creationflags = subprocess.CREATE_NEW_CONSOLE) if options['scenario'] == 'pairwise': rasterSuffix = "_current_allPairs_" + cutoffText else: rasterSuffix = "_current_allToOne_" + cutoffText currentFN = 'Circuitscape_cum_curmap.npy' currentMap = path.join(OUTCIRCUITDIR, currentFN) outputRaster = path.join(outputGDB, cfg.PREFIX + rasterSuffix) currentRaster = path.join(cfg.SCRATCHDIR, "current") try: import_npy_to_ras(currentMap, resRasClipPath, outputRaster) except: lu.dashline(1) msg = ('ERROR: Circuitscape failed. \n' 'Note: Circuitscape can only solve 2-3 million nodes' '\nper gigabyte of available RAM. The resistance ' '\nraster for the last corridor had ' + str(numResistanceNodes) + ' nodes.\n\nResistance ' 'raster values that vary by >6 orders of \nmagnitude' ' can also cause failures, as can a mismatch in ' '\ncore area and resistance raster extents.') arcpy.AddError(msg) lu.write_log(msg) exit(1) #set core areas to nodata if SETCORESTONULL: # Set core areas to NoData in current map for color ramping outputRasterND = outputRaster + '_noDataCores' outCon = arcpy.sa.SetNull( Raster(s8CoreRasClipped) > 0, Raster(outputRaster)) outCon.save(outputRasterND) gprint('\nBuilding output statistics and pyramids ' 'for centrality raster.') lu.build_stats(outputRaster) lu.build_stats(outputRasterND) # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) # Return GEOPROCESSING specific errors except arcpy.ExecuteError: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME)
def STEP8_calc_pinchpoints(): """ Maps pinch points in Linkage Mapper corridors using Circuitscape given CWD calculations from s3_calcCwds.py. """ try: lu.dashline(0) gprint('Running script ' + _SCRIPT_NAME) restartFlag = False if cfg.CWDCUTOFF < 0: cfg.CWDCUTOFF = cfg.CWDCUTOFF * -1 restartFlag = True # Restart code in progress CSPATH = lu.get_cs_path() outputGDB = path.join(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) arcpy.OverWriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.pyramid = "NONE" arcpy.env.rasterstatistics = "NONE" # set the analysis extent and cell size to that of the resistance # surface arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.snapraster = cfg.RESRAST resRaster = cfg.RESRAST arcpy.env.extent = "MINOF" minObject = arcpy.GetRasterProperties_management(resRaster, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) if rasterMin <= 0: msg = ('Error: resistance raster cannot have 0 or negative values.') lu.raise_error(msg) if cfg.DO_ADJACENTPAIRS: prevLcpShapefile = lu.get_lcp_shapefile(None, thisStep = 8) if not arcpy.Exists(prevLcpShapefile): msg = ('Cannot find an LCP shapefile from step 5. Please ' 'rerun that step and any previous ones if necessary.') lu.raise_error(msg) # Remove lcp shapefile lcpShapefile = path.join(cfg.DATAPASSDIR, "lcpLines_s8.shp") lu.delete_data(lcpShapefile) inLinkTableFile = lu.get_prev_step_link_table(step=8) linkTable = lu.load_link_table(inLinkTableFile) numLinks = linkTable.shape[0] numCorridorLinks = lu.report_links(linkTable) if numCorridorLinks == 0: lu.dashline(1) msg =('\nThere are no linkages. Bailing.') lu.raise_error(msg) if linkTable.shape[1] < 16: # If linktable has no entries from prior # centrality or pinchpint analyses extraCols = npy.zeros((numLinks, 6), dtype="float64") linkTable = linkTable[:,0:10] linkTable = npy.append(linkTable, extraCols, axis=1) linkTable[:, cfg.LTB_LCPLEN] = -1 linkTable[:, cfg.LTB_CWDEUCR] = -1 linkTable[:, cfg.LTB_CWDPATHR] = -1 linkTable[:, cfg.LTB_EFFRESIST] = -1 linkTable[:, cfg.LTB_CWDTORR] = -1 linkTable[:, cfg.LTB_CURRENT] = -1 del extraCols # set up directories for circuit and circuit mosaic grids # Create output geodatabase if not arcpy.Exists(cfg.PINCHGDB): arcpy.CreateFileGDB_management(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) mosaicRaster = path.join(cfg.CIRCUITBASEDIR, "current_mos" + tif) coresToProcess = npy.unique( linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) maxCoreNum = max(coresToProcess) del coresToProcess lu.dashline(0) coreList = linkTable[:,cfg.LTB_CORE1:cfg.LTB_CORE2+1] coreList = npy.sort(coreList) #gprint('There are ' + str(len(npy.unique(coreList))) ' core areas.') INCIRCUITDIR = cfg.CIRCUITBASEDIR OUTCIRCUITDIR = path.join(cfg.CIRCUITBASEDIR, cfg.CIRCUITOUTPUTDIR_NM) CONFIGDIR = path.join(INCIRCUITDIR, cfg.CIRCUITCONFIGDIR_NM) # Cutoff value text to append to filenames cutoffText = str(cfg.CWDCUTOFF) if cutoffText[-6:] == '000000': cutoffText = cutoffText[0:-6]+'m' elif cutoffText[-3:] == '000': cutoffText = cutoffText[0:-3]+'k' if cfg.SQUARERESISTANCES: # Square resistance values squaredRaster = path.join(cfg.SCRATCHDIR,'res_sqr') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR outRas = Raster(resRaster) * Raster(resRaster) outRas.save(squaredRaster) resRaster = squaredRaster if cfg.DO_ADJACENTPAIRS: linkLoop = 0 lu.dashline(1) gprint('Mapping pinch points in individual corridors \n' 'using Circuitscape.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(2) for x in range(0,numLinks): linkId = str(int(linkTable[x,cfg.LTB_LINKID])) if not (linkTable[x,cfg.LTB_LINKTYPE] > 0): continue linkLoop = linkLoop + 1 linkDir = path.join(cfg.SCRATCHDIR, 'link' + linkId) if restartFlag == True and path.exists(linkDir): gprint('continuing') continue restartFlag = False lu.create_dir(linkDir) start_time1 = time.clock() # source and target cores corex=int(coreList[x,0]) corey=int(coreList[x,1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) lccNormRaster = path.join(linkDir, 'lcc_norm') arcpy.env.extent = "MINOF" link = lu.get_links_from_core_pairs(linkTable, corex, corey) lcDist = float(linkTable[link,cfg.LTB_CWDIST]) # Normalized lcc rasters are created by adding cwd rasters # and subtracting the least cost distance between them. outRas = Raster(cwdRaster1) + Raster(cwdRaster2) - lcDist outRas.save(lccNormRaster) #create raster mask resMaskRaster = path.join(linkDir, 'res_mask'+tif) #create raster mask outCon = arcpy.sa.Con(Raster(lccNormRaster) <= cfg.CWDCUTOFF, 1) outCon.save(resMaskRaster) # Convert to poly. Use as mask to clip resistance raster. resMaskPoly = path.join(linkDir, 'res_mask_poly.shp') arcpy.RasterToPolygon_conversion(resMaskRaster, resMaskPoly, "NO_SIMPLIFY") arcpy.env.extent = resMaskPoly # Includes 0 values in some cases with CP LI model if tif # so using ESRI Grid format resClipRasterMasked = path.join(linkDir, 'res_clip_m') # Extract masked resistance raster. # Needs to be float to get export to npy to work. outRas = arcpy.sa.ExtractByMask(resRaster, resMaskPoly) + 0.0 outRas.save(resClipRasterMasked) resNpyFN = 'resistances_link_' + linkId + '.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy(resClipRasterMasked, resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) gwarn('Warning:') gwarn('Circuitscape can only solve 2-3 million nodes') gwarn('per gigabyte of available RAM. \nTotal physical RAM' ' on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~'+ str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(2) corePairRaster = path.join(linkDir, 'core_pairs'+tif) arcpy.env.extent = resClipRasterMasked # Next result needs to be floating pt for numpy export outCon = arcpy.sa.Con(Raster(cwdRaster1) == 0, corex, arcpy.sa.Con(Raster(cwdRaster2) == 0, corey + 0.0)) outCon.save(corePairRaster) coreNpyFN = 'cores_link_' + linkId + '.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy(corePairRaster, coreNpyFile) arcpy.env.extent = "MINOF" # Set circuitscape options and call options = lu.setCircuitscapeOptions() if cfg.WRITE_VOLT_MAPS == True: options['write_volt_maps']=True options['habitat_file'] = resNpyFile # if int(linkId) > 2: # options['habitat_file'] = 'c:\\test.dummy' options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero']=True outputFN = 'Circuitscape_link' + linkId + '.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) if numElements > 250000: options['print_timings']=True configFN = 'pinchpoint_config' + linkId + '.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('Processing link ID #' + str(linkId) + '. Resistance map' ' has ' + str(int(numResistanceNodes)) + ' nodes.') memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): print_failure(numResistanceNodes, memFlag, 10) numElements, numNodes = export_ras_to_npy( resClipRasterMasked,resNpyFile) memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): msg = ('\nCircuitscape failed. See error information above.') arcpy.AddError(msg) lu.write_log(msg) exit(1) # Either set core areas to nodata in current map or # divide each by its radius currentRaster = path.join(linkDir, "current" + tif) import_npy_to_ras(currentMap,corePairRaster,currentRaster) if cfg.WRITE_VOLT_MAPS == True: voltFN = ('Circuitscape_link' + linkId + '_voltmap_' + str(corex) + '_'+str(corey) + '.npy') voltMap = path.join(OUTCIRCUITDIR, voltFN) voltRaster = path.join(outputGDB, cfg.PREFIX + "_voltMap_"+ str(corex) + '_'+str(corey)) import_npy_to_ras(voltMap,corePairRaster,voltRaster) gprint('Building output statistics and pyramids ' 'for voltage raster\n') lu.build_stats(voltRaster) arcpy.env.extent = currentRaster if SETCORESTONULL: # Set core areas to NoData in current map for color ramping currentRaster2 = currentRaster + '2' + tif outCon = arcpy.sa.Con(arcpy.sa.IsNull(Raster (corePairRaster)), Raster(currentRaster)) outCon.save(currentRaster2) currentRaster = currentRaster2 arcpy.env.extent = "MAXOF" if linkLoop == 1: lu.delete_data(mosaicRaster) @retry(10) def copyRas2(): arcpy.CopyRaster_management(currentRaster, mosaicRaster) copyRas2() else: @retry(10) def mosaicRas(): arcpy.Mosaic_management(currentRaster, mosaicRaster, "MAXIMUM", "MATCH") mosaicRas() resistancesFN = ('Circuitscape_link' + linkId + '_resistances_3columns.out') resistancesFile = path.join(OUTCIRCUITDIR,resistancesFN) resistances = npy.loadtxt(resistancesFile, dtype = 'Float64', comments='#') resistance = float(str(arcpy.env.cellSize)) * resistances[2] linkTable[link,cfg.LTB_EFFRESIST] = resistance # Ratio if not cfg.SQUARERESISTANCES: linkTable[link,cfg.LTB_CWDTORR] = (linkTable[link, cfg.LTB_CWDIST] / linkTable[link,cfg.LTB_EFFRESIST]) # Clean up if cfg.SAVE_TEMP_CIRCUIT_FILES == False: lu.delete_file(coreNpyFile) coreNpyBase, extension = path.splitext(coreNpyFile) lu.delete_data(coreNpyBase + '.hdr') lu.delete_file(resNpyFile) resNpyBase, extension = path.splitext(resNpyFile) lu.delete_data(resNpyBase + '.hdr') lu.delete_file(currentMap) curMapBase, extension = path.splitext(currentMap) lu.delete_data(curMapBase + '.hdr') lu.delete_data(currentRaster) lu.clean_out_workspace(linkDir) lu.delete_dir(linkDir) gprint('Finished with link ID #' + str(linkId) + '. ' + str(linkLoop) + ' out of ' + str(numCorridorLinks) + ' links have been processed.') start_time1 = lu.elapsed_time(start_time1) outputRaster = path.join(outputGDB, cfg.PREFIX + "_current_adjacentPairs_" + cutoffText) lu.delete_data(outputRaster) @retry(10) def copyRas(): arcpy.CopyRaster_management(mosaicRaster, outputRaster) copyRas() gprint('Building output statistics and pyramids ' 'for corridor pinch point raster\n') lu.build_stats(outputRaster) finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=5, thisStep=8) linkTableFile = path.join(cfg.DATAPASSDIR, "linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFile, inLinkTableFile) linkTableFinalFile = path.join(cfg.OUTPUTDIR, cfg.PREFIX + "_linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFinalFile, inLinkTableFile) gprint('Copy of linkTable written to '+ linkTableFinalFile) #fixme: update sticks? gprint('Creating shapefiles with linework for links.') lu.write_link_maps(linkTableFinalFile, step=8) # Copy final link maps to gdb. lu.copy_final_link_maps(step=8) lu.delete_data(mosaicRaster) if not cfg.DO_ALLPAIRS: # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) return lu.dashline(1) gprint('Mapping global pinch points among all\n' 'core area pairs using Circuitscape.') if cfg.ALL_PAIR_SCENARIO=='pairwise': gprint('Circuitscape will be run in PAIRWISE mode.') else: gprint('Circuitscape will be run in ALL-TO-ONE mode.') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST S8CORE_RAS = "s8core_ras" s8CoreRasPath = path.join(cfg.SCRATCHDIR,S8CORE_RAS) arcpy.FeatureToRaster_conversion(cfg.COREFC, cfg.COREFN, s8CoreRasPath, arcpy.env.cellSize) binaryCoreRaster = path.join(cfg.SCRATCHDIR,"core_ras_bin") # The following commands cause file lock problems on save. using gp # instead. # outCon = arcpy.sa.Con(S8CORE_RAS, 1, "#", "VALUE > 0") # outCon.save(binaryCoreRaster) # gp.Con_sa(s8CoreRasPath, 1, binaryCoreRaster, "#", "VALUE > 0") outCon = arcpy.sa.Con(Raster(s8CoreRasPath) > 0, 1) outCon.save(binaryCoreRaster) s5corridorRas = path.join(cfg.OUTPUTGDB,cfg.PREFIX + "_corridors") if not arcpy.Exists(s5corridorRas): s5corridorRas = path.join(cfg.OUTPUTGDB,cfg.PREFIX + "_lcc_mosaic_int") outCon = arcpy.sa.Con(Raster(s5corridorRas) <= cfg.CWDCUTOFF, Raster( resRaster), arcpy.sa.Con(Raster( binaryCoreRaster) > 0, Raster(resRaster))) resRasClipPath = path.join(cfg.SCRATCHDIR,'res_ras_clip') outCon.save(resRasClipPath) arcpy.env.cellSize = resRasClipPath arcpy.env.extent = resRasClipPath s8CoreRasClipped = s8CoreRasPath + '_c' # Produce core raster with same extent as clipped resistance raster # added to ensure correct data type- nodata values were positive for # cores otherwise outCon = arcpy.sa.Con(arcpy.sa.IsNull(Raster(s8CoreRasPath)), -9999, Raster(s8CoreRasPath)) outCon.save(s8CoreRasClipped) resNpyFN = 'resistances.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy(resRasClipPath,resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) gwarn('Warning:') gwarn('Circuitscape can only solve 2-3 million nodes') gwarn('per gigabyte of available RAM. \nTotal physical RAM ' 'on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~'+ str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(0) coreNpyFN = 'cores.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy(s8CoreRasClipped,coreNpyFile) arcpy.env.extent = "MINOF" options = lu.setCircuitscapeOptions() options['scenario']=cfg.ALL_PAIR_SCENARIO options['habitat_file'] = resNpyFile options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero']=True outputFN = 'Circuitscape.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) options['print_timings']=True configFN = 'pinchpoint_allpair_config.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('\nResistance map has ' + str(int(numResistanceNodes)) + ' nodes.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(0) call_circuitscape(CSPATH, outConfigFile) # test = subprocess.call([CSPATH, outConfigFile], # creationflags = subprocess.CREATE_NEW_CONSOLE) if options['scenario']=='pairwise': rasterSuffix = "_current_allPairs_" + cutoffText else: rasterSuffix = "_current_allToOne_" + cutoffText currentFN = 'Circuitscape_cum_curmap.npy' currentMap = path.join(OUTCIRCUITDIR, currentFN) outputRaster = path.join(outputGDB, cfg.PREFIX + rasterSuffix) currentRaster = path.join(cfg.SCRATCHDIR, "current") try: import_npy_to_ras(currentMap,resRasClipPath,outputRaster) except: lu.dashline(1) msg = ('ERROR: Circuitscape failed. \n' 'Note: Circuitscape can only solve 2-3 million nodes' '\nper gigabyte of available RAM. The resistance ' '\nraster for the last corridor had ' + str(numResistanceNodes) + ' nodes.\n\nResistance ' 'raster values that vary by >6 orders of \nmagnitude' ' can also cause failures, as can a mismatch in ' '\ncore area and resistance raster extents.') arcpy.AddError(msg) lu.write_log(msg) exit(1) #set core areas to nodata if SETCORESTONULL: # Set core areas to NoData in current map for color ramping outputRasterND = outputRaster + '_noDataCores' outCon = arcpy.sa.SetNull(Raster(s8CoreRasClipped) > 0, Raster(outputRaster)) outCon.save(outputRasterND) gprint('\nBuilding output statistics and pyramids ' 'for centrality raster.') lu.build_stats(outputRaster) lu.build_stats(outputRasterND) # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) # Return GEOPROCESSING specific errors except arcpy.ExecuteError: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME)
def STEP3_calc_cwds(): """Calculates cost-weighted distances from each core area. Uses bounding circles around source and target cores to limit extent of cwd calculations and speed computation. """ try: lu.dashline(1) gprint('Running script ' + _SCRIPT_NAME) lu.dashline(0) # Super secret setting to re-start failed run. Enter 'RESTART' as the # Name of the pairwise distance table in step 2, and uncheck step 2. # We can eventually place this in a .ini file. rerun = False if cfg.S2EUCDISTFILE != None: if cfg.S2EUCDISTFILE.lower() == "restart": rerun = True # if cfg.TMAXCWDIST is None: # gprint('NOT using a maximum cost-weighted distance.') # else: # gprint('Max cost-weighted distance for CWD calcs set ' # 'to ' + str(cfg.TMAXCWDIST) + '\n') if (cfg.BUFFERDIST) is not None: gprint('Bounding circles plus a buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units will ' 'be used \n to limit extent of cost distance ' 'calculations.') elif cfg.TOOL <> cfg.TOOL_CC: gprint('NOT using bounding circles in cost distance ' 'calculations.') # set the analysis extent and cell size # So we don't extract rasters that go beyond extent of original raster if arcpy: arcpy.env.cellSize = cfg.RESRAST arcpy.env.extent="MINOF" else: gp.cellSize = gp.Describe(cfg.RESRAST).MeanCellHeight gp.Extent = "MINOF" gp.mask = cfg.RESRAST if arcpy: arcpy.env.overwriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR else: gp.OverwriteOutput = True gp.workspace = cfg.SCRATCHDIR gp.scratchWorkspace = cfg.ARCSCRATCHDIR # Load linkTable (created in previous script) linkTableFile = lu.get_prev_step_link_table(step=3) linkTable = lu.load_link_table(linkTableFile) lu.report_links(linkTable) # Identify cores to map from LinkTable coresToMap = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) numCoresToMap = len(coresToMap) if numCoresToMap < 3: # No need to check for intermediate cores, because there aren't any cfg.S3DROPLCCSic = False else: cfg.S3DROPLCCSic = cfg.S3DROPLCCS gprint('\nNumber of core areas to connect: ' + str(numCoresToMap)) if rerun: # If picking up a failed run, make sure needed files are there lu.dashline(1) gprint ('\n****** RESTART MODE ENABLED ******\n') gprint ('**** NOTE: This mode picks up step 3 where a\n' 'previous run left off due to a crash or user\n' 'abort. It assumes you are using the same input\n' 'data used in the terminated run.****\n') lu.dashline(0) lu.snooze(10) savedLinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") coreListFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") if not path.exists(savedLinkTableFile) or not path.exists( coreListFile): gprint('No partial results file found from previous ' 'stopped run. Starting run from beginning.\n') lu.dashline(0) rerun = False # If picking up a failed run, use old folders if not rerun: startIndex = 0 if cfg.TOOL <> cfg.TOOL_CC: lu.make_cwd_paths(max(coresToMap)) # Set up cwd directories # make a feature layer for input cores to select from gp.MakeFeatureLayer(cfg.COREFC, cfg.FCORES) # Drop links that are too long gprint('\nChecking for corridors that are too long to map.') DISABLE_LEAST_COST_NO_VAL = False linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, 0, cfg.MAXCOSTDIST, 0, DISABLE_LEAST_COST_NO_VAL) # ------------------------------------------------------------------ # Bounding boxes if (cfg.BUFFERDIST) is not None: # create bounding boxes around cores start_time = time.clock() # lu.dashline(1) gprint('Calculating bounding boxes for core areas.') extentBoxList = npy.zeros((0,5), dtype='float32') for x in range(len(coresToMap)): core = coresToMap[x] boxCoords = lu.get_extent_box_coords(core) extentBoxList = npy.append(extentBoxList, boxCoords, axis=0) gprint('\nDone calculating bounding boxes.') start_time = lu.elapsed_time(start_time) # lu.dashline() # Bounding circle code if cfg.BUFFERDIST is not None: # Make a set of circles encompassing core areas we'll be connecting start_time = time.clock() gprint('Calculating bounding circles around potential' ' corridors.') # x y corex corey radius- stores data for bounding circle centroids boundingCirclePointArray = npy.zeros((0,5), dtype='float32') circleList = npy.zeros((0,3), dtype='int32') numLinks = linkTable.shape[0] for x in range(0, numLinks): if ((linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_CORR) or (linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_KEEP)): # if it's a valid corridor link linkId = int(linkTable[x,cfg.LTB_LINKID]) # fixme- this code is clumsy- can trim down cores = npy.zeros((1,3), dtype='int32') cores[0,:] = npy.sort([0, linkTable[x,cfg.LTB_CORE1], linkTable[x,cfg.LTB_CORE2]]) corex = cores[0,1] corey = cores[0,2] cores[0,0] = linkId ################### foundFlag = False for y in range(0,len(circleList)): # clumsy if (circleList[y,1] == corex and circleList[y,2] == corey): foundFlag = True if not foundFlag: circlePointData = ( lu.get_bounding_circle_data(extentBoxList, corex, corey, cfg.BUFFERDIST)) boundingCirclePointArray = ( npy.append(boundingCirclePointArray, circlePointData, axis=0)) # keep track of which cores we draw bounding circles # around circleList = npy.append(circleList, cores, axis=0) gprint('\nCreating bounding circles using buffer ' 'analysis.') dir, BNDCIRCENS = path.split(cfg.BNDCIRCENS) lu.make_points(cfg.SCRATCHDIR, boundingCirclePointArray, BNDCIRCENS) lu.delete_data(cfg.BNDCIRS) gp.buffer_analysis(cfg.BNDCIRCENS, cfg.BNDCIRS, "radius") gp.deletefield (cfg.BNDCIRS, "BUFF_DIST") gprint('Successfully created bounding circles around ' 'potential corridors using \na buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.') start_time = lu.elapsed_time(start_time) gprint('Reducing global processing area using bounding ' 'circle plus buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.\n') extentBoxList = npy.zeros((0,5),dtype='float32') boxCoords = lu.get_extent_box_coords() extentBoxList = npy.append(extentBoxList,boxCoords,axis=0) extentBoxList[0,0] = 0 boundingCirclePointArray = npy.zeros((0,5),dtype='float32') circlePointData=lu.get_bounding_circle_data(extentBoxList, 0, 0, cfg.BUFFERDIST) dir, BNDCIRCEN = path.split(cfg.BNDCIRCEN) lu.make_points(cfg.SCRATCHDIR, circlePointData, BNDCIRCEN) lu.delete_data(cfg.BNDCIR) gp.buffer_analysis(cfg.BNDCIRCEN, cfg.BNDCIR, "radius") gprint('Extracting raster....') cfg.BOUNDRESIS = cfg.BOUNDRESIS + tif lu.delete_data(cfg.BOUNDRESIS) count = 0 statement = ( 'gp.ExtractByMask_sa(cfg.RESRAST, cfg.BNDCIR, cfg.BOUNDRESIS)') while True: try: exec statement randomerror() except: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec statement else: break gprint('\nReduced resistance raster extracted using ' 'bounding circle.') else: #if not using bounding circles, just go with resistance raster. cfg.BOUNDRESIS = cfg.RESRAST # --------------------------------------------------------------------- # Rasterize core areas to speed cost distance calcs # lu.dashline(1) gprint("Creating core area raster.") gp.SelectLayerByAttribute(cfg.FCORES, "CLEAR_SELECTION") if arcpy: arcpy.env.cellSize = cfg.BOUNDRESIS arcpy.env.extent = cfg.BOUNDRESIS else: gp.cellSize = gp.Describe(cfg.BOUNDRESIS).MeanCellHeight gp.extent = gp.Describe(cfg.BOUNDRESIS).extent if rerun: # saved linktable replaces the one now in memory linkTable = lu.load_link_table(savedLinkTableFile) coresToMapSaved = npy.loadtxt(coreListFile, dtype='Float64', comments='#', delimiter=',') startIndex = coresToMapSaved[0] # Index of core where we left off del coresToMapSaved gprint ('\n****** Re-starting run at core area number ' + str(int(coresToMap[startIndex]))+ ' ******\n') lu.dashline(0) if arcpy: arcpy.env.extent = "MINOF" else: gp.extent = "MINOF" #---------------------------------------------------------------------- # Loop through cores, do cwd calcs for each if cfg.TOOL == cfg.TOOL_CC: gprint("\nMapping least-cost paths.\n") else: gprint("\nStarting cost distance calculations.\n") lcpLoop = 0 failures = 0 x = startIndex endIndex = len(coresToMap) linkTableMod = linkTable.copy() while x < endIndex: startTime1 = time.clock() # Modification of linkTable in function was causing problems. so # make a copy: linkTablePassed = linkTableMod.copy() (linkTableReturned, failures, lcpLoop) = do_cwd_calcs(x, linkTablePassed, coresToMap, lcpLoop, failures) if failures == 0: # If iteration was successful, continue with next core linkTableMod = linkTableReturned sourceCore = int(coresToMap[x]) gprint('Done with all calculations for core ID #' + str(sourceCore) + '. ' + str(int(x + 1)) + ' of ' + str(endIndex) + ' cores have been processed.') start_time = lu.elapsed_time(startTime1) outlinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.write_link_table(linkTableMod, outlinkTableFile) # Increment loop counter x = x + 1 else: # If iteration failed, try again after a wait period delay_restart(failures) #---------------------------------------------------------------------- linkTable = linkTableMod # reinstate temporarily disabled links rows = npy.where(linkTable[:,cfg.LTB_LINKTYPE] > 1000) linkTable[rows,cfg.LTB_LINKTYPE] = (linkTable[rows,cfg.LTB_LINKTYPE] - 1000) # Drop links that are too long DISABLE_LEAST_COST_NO_VAL = True linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, cfg.MINEUCDIST, cfg.MAXCOSTDIST, cfg.MINCOSTDIST, DISABLE_LEAST_COST_NO_VAL) # Write link table file outlinkTableFile = lu.get_this_step_link_table(step=3) gprint('Updating ' + outlinkTableFile) lu.write_link_table(linkTable, outlinkTableFile) linkTableLogFile = path.join(cfg.LOGDIR, "linkTable_s3.csv") lu.write_link_table(linkTable, linkTableLogFile) start_time = time.clock() gprint('Creating shapefiles with linework for links...') try: lu.write_link_maps(outlinkTableFile, step=3) except: lu.write_link_maps(outlinkTableFile, step=3) start_time = lu.elapsed_time(start_time) gprint('\nIndividual cost-weighted distance layers written ' 'to "cwd" directory. \n') gprint(outlinkTableFile + '\n updated with cost-weighted distances between core areas.') #Clean up temporary files for restart code tempFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") lu.delete_file(tempFile) tempFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.delete_file(tempFile) # Check if climate tool is calling linkage mapper if cfg.TOOL == cfg.TOOL_CC: coreList = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) for core in coreList: cwdRaster = lu.get_cwd_path(int(core)) back_rast = cwdRaster.replace("cwd_", "back_") lu.delete_data(back_rast) # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME) return