def func_out(*args, **kwargs): """Run and if necessary retry running function.""" exception = None for i in range(self.tries): try: return func_in(*args, **kwargs) except self.exceptions as func_error: tbobj = sys.exc_info()[2] # Get the traceback object # tbinfo contains the error's line number and the code tbinfo = traceback.format_tb(tbobj)[1] line = tbinfo.split(", ")[1] filename = tbinfo.split(", ")[0] filename = filename.rsplit("File ")[1] lu.warn('------------------------------------------------' '--') msg = ("The following error is being reported " "on " + line + " of " + filename + ":") lu.warn(msg) lu.warn(str(func_error)) lu.print_drive_warning() delaytime = self.delay * 10 * (i + 1) lu.warn("Will try again. ") lu.warn('---------RETRY #' + str(i + 1) + ' OUT OF ' + str(self.tries) + ' IN ' + str(delaytime) + ' SECONDS---------\n') lu.snooze(delaytime) exception = func_error # If no success after tries, raise last exception raise exception
def run_analysis(): """Run Climate Linkage Mapper analysis.""" import cc_grass_cwd # Cannot import until configured cc_copy_inputs() # Clip inputs and create project area raster # Get zonal statistics for cores and climate lm_util.gprint("\nCALCULATING ZONAL STATISTICS FROM CLIMATE RASTER") climate_stats = arcpy.sa.ZonalStatisticsAsTable(cc_env.prj_core_fc, cc_env.core_fld, cc_env.prj_climate_rast, "zstats", "DATA", "ALL") # Create core pairings table and limit based upon climate threshold core_pairings = create_pair_tbl(climate_stats) # Generate link table, calculate CWD and run Linkage Mapper if int(arcpy.GetCount_management(core_pairings).getOutput(0)) == 0: lm_util.warn("\nNo core pairs within climate threshold. " "Program will end") else: # Process pairings and generate link table grass_cores = process_pairings(core_pairings) if not grass_cores: lm_util.warn("\nNo core pairs within Euclidean distances. " "Progam will end") else: # Create CWD using Grass cc_grass_cwd.grass_cwd(grass_cores) # Run Linkage Mapper lm_util.gprint("\nRUNNING LINKAGE MAPPER " "TO CREATE CLIMATE CORRIDORS") lm_master.lm_master()
def fn(*args, **kwargs): exception = None for _ in range(self.tries): try: return f(*args, **kwargs) except self.exceptions, e: import traceback tb = sys.exc_info()[2] # get the traceback object # tbinfo contains the error's line number and the code tbinfo = traceback.format_tb(tb)[1] line = tbinfo.split(", ")[1] filename = tbinfo.split(", ")[0] filename = filename.rsplit("File ")[1] lu.warn( '--------------------------------------------------') msg = ("The following error is being reported " "on " + line + " of " + filename + ":") lu.warn(msg) # lu.write_log(msg) lu.warn(str(e)) # lu.write_log(str(e)) lu.print_drive_warning() delaytime = self.delay * 10 * (_ + 1) lu.warn("Will try again. ") lu.warn('---------RETRY #' + str(_ + 1) + ' OUT OF ' + str(self.tries) + ' IN ' + str(delaytime) + ' SECONDS---------\n') lu.snooze(delaytime) exception = e
def grass_dir_setup(): """Create new folder for GRASS workspace""" gisdbase = os.path.join(cc_env.proj_dir, "gwksp") # Remove GRASS directory if it exists if not cc_util.remove_grass_wkspc(gisdbase): lm_util.warn("\nCannot delete GRASS workspace from earlier" " run." "\nPlease choose a new project directory.") raise Exception("Cannot delete GRASS workspace: " + gisdbase)
def grass_cwd(core_list): """Creating CWD and Back rasters using GRASS r.walk function""" gisdbase = os.path.join(cc_env.proj_dir, "gwksp") ccr_grassrc = os.path.join(cc_env.proj_dir, "ccr_grassrc") climate_asc = os.path.join(cc_env.out_dir, "cc_climate.asc") resist_asc = os.path.join(cc_env.out_dir, "cc_resist.asc") core_asc = os.path.join(cc_env.out_dir, "cc_cores.asc") climate_lyr = "climate" resist_lyr = "resist" core_lyr = "cores" try: lm_util.gprint("\nRUNNING GRASS TO CREATE COST-WEIGHTED DISTANCE " "RASTERS") # Convert input GRID rasters to ASCII lm_util.gprint("Converting ARCINFO GRID rasters to ASCII") arcpy.RasterToASCII_conversion(cc_env.prj_climate_rast, climate_asc) arcpy.RasterToASCII_conversion(cc_env.prj_resist_rast, resist_asc) arcpy.RasterToASCII_conversion(cc_env.prj_core_rast, core_asc) # Create resource file and setup workspace start_path = os.environ["PATH"] os.environ["PATH"] = cc_env.gpath write_grassrc(ccr_grassrc, gisdbase) setup_wrkspace(gisdbase, ccr_grassrc, climate_asc) # Make cwd folder for Linkage Mapper lm_util.make_cwd_paths(max(core_list)) # Import files into GRASS lm_util.gprint("Importing raster files into GRASS") run_grass_cmd("r.in.gdal", input=climate_asc, output=climate_lyr) run_grass_cmd("r.in.gdal", input=resist_asc, output=resist_lyr) run_grass_cmd("r.in.gdal", input=core_asc, output=core_lyr) # Generate CWD and Back rasters gen_cwd_back(core_list, climate_lyr, resist_lyr, core_lyr) except Exception: raise finally: os.environ["PATH"] = start_path if not cc_util.remove_grass_wkspc(gisdbase): lm_util.warn("Unable to delete temporary GRASS folder. " "Program will contine.") cc_util.delete_features( [climate_asc, resist_asc, core_asc, ccr_grassrc])
def calc_lccs(normalize): try: if normalize: mosaicBaseName = "_corridors" writeTruncRaster = cfg.WRITETRUNCRASTER outputGDB = cfg.OUTPUTGDB SAVENORMLCCS = cfg.SAVENORMLCCS else: mosaicBaseName = "_NON_NORMALIZED_corridors" SAVENORMLCCS = False outputGDB = cfg.EXTRAGDB writeTruncRaster = False lu.dashline(1) gprint('Running script ' + _SCRIPT_NAME) linkTableFile = lu.get_prev_step_link_table(step=5) arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.compression = "NONE" if cfg.MAXEUCDIST is not None: gprint('Max Euclidean distance between cores') gprint('for linkage mapping set to ' + str(cfg.MAXEUCDIST)) if cfg.MAXCOSTDIST is not None: gprint('Max cost-weighted distance between cores') gprint('for linkage mapping set to ' + str(cfg.MAXCOSTDIST)) # set the analysis extent and cell size to that of the resistance # surface arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = arcpy.Describe(cfg.RESRAST).MeanCellHeight arcpy.env.snapRaster = cfg.RESRAST arcpy.env.mask = cfg.RESRAST linkTable = lu.load_link_table(linkTableFile) numLinks = linkTable.shape[0] numCorridorLinks = lu.report_links(linkTable) if numCorridorLinks == 0: lu.dashline(1) msg =('\nThere are no corridors to map. Bailing.') lu.raise_error(msg) if not cfg.STEP3 and not cfg.STEP4: # re-check for links that are too long or in case script run out of # sequence with more stringent settings gprint('Double-checking for corridors that are too long to map.') DISABLE_LEAST_COST_NO_VAL = True linkTable,numDroppedLinks = lu.drop_links( linkTable, cfg.MAXEUCDIST, cfg.MINEUCDIST, cfg.MAXCOSTDIST, cfg.MINCOSTDIST, DISABLE_LEAST_COST_NO_VAL) # Added to try to speed up: arcpy.env.pyramid = "NONE" arcpy.env.rasterStatistics = "NONE" # set up directories for normalized lcc and mosaic grids dirCount = 0 gprint("Creating output folder: " + cfg.LCCBASEDIR) lu.delete_dir(cfg.LCCBASEDIR) arcpy.CreateFolder_management(path.dirname(cfg.LCCBASEDIR), path.basename(cfg.LCCBASEDIR)) arcpy.CreateFolder_management(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM) clccdir = path.join(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM) gprint("") if normalize: gprint('Normalized least-cost corridors will be written ' 'to ' + clccdir + '\n') PREFIX = cfg.PREFIX # Add CWD layers for core area pairs to produce NORMALIZED LCC layers numGridsWritten = 0 coreList = linkTable[:,cfg.LTB_CORE1:cfg.LTB_CORE2+1] coreList = npy.sort(coreList) x = 0 linkCount = 0 endIndex = numLinks while x < endIndex: if (linkTable[x, cfg.LTB_LINKTYPE] < 1): # If not a valid link x = x + 1 continue linkCount = linkCount + 1 start_time = time.clock() linkId = str(int(linkTable[x, cfg.LTB_LINKID])) # source and target cores corex=int(coreList[x,0]) corey=int(coreList[x,1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) if not arcpy.Exists(cwdRaster1): msg =('\nError: cannot find cwd raster:\n' + cwdRaster1) if not arcpy.Exists(cwdRaster2): msg =('\nError: cannot find cwd raster:\n' + cwdRaster2) lu.raise_error(msg) lccNormRaster = path.join(clccdir, str(corex) + "_" + str(corey))# + ".tif") arcpy.env.extent = "MINOF" link = lu.get_links_from_core_pairs(linkTable, corex, corey) offset = 10000 # Normalized lcc rasters are created by adding cwd rasters and # subtracting the least cost distance between them. lcDist = (float(linkTable[link,cfg.LTB_CWDIST]) - offset) if normalize: statement = ('outras = arcpy.sa.Raster(cwdRaster1) ' '+ arcpy.sa.Raster(cwdRaster2) - lcDist; ' 'outras.save(lccNormRaster)') else: statement = ('outras = arcpy.sa.Raster(cwdRaster1) ' '+ arcpy.sa.Raster(cwdRaster2); ' 'outras.save(lccNormRaster)') count = 0 while True: try: exec(statement) except Exception: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec(statement) else: break if normalize: try: minObject = arcpy.GetRasterProperties_management(lccNormRaster, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) except Exception: lu.warn('\n------------------------------------------------') lu.warn('WARNING: Raster minimum check failed in step 5. \n' 'This may mean the output rasters are corrupted. Please \n' 'be sure to check for valid rasters in '+ outputGDB) rasterMin = 0 tolerance = (float(arcpy.env.cellSize) * -10) if rasterMin < tolerance: lu.dashline(1) msg = ('WARNING: Minimum value of a corridor #' + str(x+1) + ' is much less than zero ('+str(rasterMin)+').' '\nThis could mean that BOUNDING CIRCLE BUFFER DISTANCES ' 'were too small and a corridor passed outside of a ' 'bounding circle, or that a corridor passed outside of the ' 'resistance map. \n') lu.warn(msg) arcpy.env.extent = cfg.RESRAST mosaicDir = path.join(cfg.LCCBASEDIR,'mos'+str(x+1)) lu.create_dir(mosaicDir) mosFN = 'mos'#.tif' change and move mosaicRaster = path.join(mosaicDir,mosFN) if numGridsWritten == 0 and dirCount == 0: #If this is the first grid then copy rather than mosaic arcpy.CopyRaster_management(lccNormRaster, mosaicRaster) else: statement = ( 'arcpy.MosaicToNewRaster_management(' 'input_rasters=";".join([lccNormRaster, ' 'lastMosaicRaster]), output_location=mosaicDir, ' 'raster_dataset_name_with_extension=mosFN, ' 'pixel_type="32_BIT_FLOAT", cellsize=arcpy.env.cellSize, ' 'number_of_bands="1", mosaic_method="MINIMUM")') count = 0 while True: try: lu.write_log('Executing mosaic for link #'+str(linkId)) exec(statement) lu.write_log('Done with mosaic.') except Exception: count,tryAgain = lu.retry_arc_error(count,statement) lu.delete_data(mosaicRaster) lu.delete_dir(mosaicDir) # Try a new directory mosaicDir = path.join(cfg.LCCBASEDIR,'mos'+str(x+1)+ '_' + str(count)) lu.create_dir(mosaicDir) mosaicRaster = path.join(mosaicDir,mosFN) if not tryAgain: exec(statement) else: break endTime = time.clock() processTime = round((endTime - start_time), 2) if normalize == True: printText = "Normalized and mosaicked " else: printText = "Mosaicked NON-normalized " gprint(printText + "corridor for link ID #" + str(linkId) + " connecting core areas " + str(corex) + " and " + str(corey)+ " in " + str(processTime) + " seconds. " + str(int(linkCount)) + " out of " + str(int(numCorridorLinks)) + " links have been " "processed.") # temporarily disable links in linktable - don't want to mosaic # them twice for y in range (x+1,numLinks): corex1 = int(coreList[y,0]) corey1 = int(coreList[y,1]) if corex1 == corex and corey1 == corey: linkTable[y,cfg.LTB_LINKTYPE] = ( linkTable[y,cfg.LTB_LINKTYPE] + 1000) elif corex1==corey and corey1==corex: linkTable[y,cfg.LTB_LINKTYPE] = ( linkTable[y,cfg.LTB_LINKTYPE] + 1000) numGridsWritten = numGridsWritten + 1 if not SAVENORMLCCS: lu.delete_data(lccNormRaster) lu.delete_dir(clccdir) lu.create_dir(clccdir) else: if numGridsWritten == 100: # We only write up to 100 grids to any one folder # because otherwise Arc slows to a crawl dirCount = dirCount + 1 numGridsWritten = 0 clccdir = path.join(cfg.LCCBASEDIR, cfg.LCCNLCDIR_NM + str(dirCount)) gprint("Creating output folder: " + clccdir) arcpy.CreateFolder_management(cfg.LCCBASEDIR, path.basename(clccdir)) if numGridsWritten > 1 or dirCount > 0: lu.delete_data(lastMosaicRaster) lu.delete_dir(path.dirname(lastMosaicRaster)) lastMosaicRaster = mosaicRaster x = x + 1 #rows that were temporarily disabled rows = npy.where(linkTable[:,cfg.LTB_LINKTYPE]>1000) linkTable[rows,cfg.LTB_LINKTYPE] = ( linkTable[rows,cfg.LTB_LINKTYPE] - 1000) # --------------------------------------------------------------------- # Create output geodatabase if not arcpy.Exists(outputGDB): arcpy.CreateFileGDB_management(cfg.OUTPUTDIR, path.basename(outputGDB)) arcpy.env.workspace = outputGDB arcpy.env.pyramid = "NONE" arcpy.env.rasterStatistics = "NONE" # --------------------------------------------------------------------- # convert mosaic raster to integer intRaster = path.join(outputGDB,PREFIX + mosaicBaseName) statement = ('outras = arcpy.sa.Int(arcpy.sa.Raster(mosaicRaster) ' '- offset + 0.5); ' 'outras.save(intRaster)') count = 0 while True: try: exec(statement) except Exception: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec(statement) else: break # --------------------------------------------------------------------- if writeTruncRaster: # ----------------------------------------------------------------- # Set anything beyond cfg.CWDTHRESH to NODATA. truncRaster = (outputGDB + '\\' + PREFIX + mosaicBaseName + '_truncated_at_' + lu.cwd_cutoff_str(cfg.CWDTHRESH)) statement = ('outRas = arcpy.sa.Raster(intRaster)' '* (arcpy.sa.Con(arcpy.sa.Raster(intRaster) ' '<= cfg.CWDTHRESH, 1)); ' 'outRas.save(truncRaster)') count = 0 while True: try: exec(statement) except Exception: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec(statement) else: break # --------------------------------------------------------------------- # Check for unreasonably low minimum NLCC values try: mosaicGrid = path.join(cfg.LCCBASEDIR,'mos') # Copy to grid to test arcpy.CopyRaster_management(mosaicRaster, mosaicGrid) minObject = arcpy.GetRasterProperties_management(mosaicGrid, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) except Exception: lu.warn('\n------------------------------------------------') lu.warn('WARNING: Raster minimum check failed in step 5. \n' 'This may mean the output rasters are corrupted. Please \n' 'be sure to check for valid rasters in '+ outputGDB) rasterMin = 0 tolerance = (float(arcpy.env.cellSize) * -10) if rasterMin < tolerance: lu.dashline(1) msg = ('WARNING: Minimum value of mosaicked corridor map is ' 'much less than zero ('+str(rasterMin)+').' '\nThis could mean that BOUNDING CIRCLE BUFFER DISTANCES ' 'were too small and a corridor passed outside of a ' 'bounding circle, or that a corridor passed outside of the ' 'resistance map. \n') lu.warn(msg) gprint('\nWriting final LCP maps...') if cfg.STEP4: finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=4, thisStep=5) elif cfg.STEP3: finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=3, thisStep=5) else: # Don't know if step 4 was run, since this is started at step 5. # Use presence of previous linktable files to figure this out. # Linktable name includes step number. prevLinkTableFile = lu.get_prev_step_link_table(step=5) prevStepInd = len(prevLinkTableFile) - 5 lastStep = prevLinkTableFile[prevStepInd] finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep, thisStep=5) outlinkTableFile = lu.get_this_step_link_table(step=5) gprint('Updating ' + outlinkTableFile) lu.write_link_table(linkTable, outlinkTableFile) linkTableLogFile = path.join(cfg.LOGDIR, "linkTable_s5.csv") lu.write_link_table(linkTable, linkTableLogFile) linkTableFinalFile = path.join(cfg.OUTPUTDIR, PREFIX + "_linkTable_s5.csv") lu.write_link_table(finalLinkTable, linkTableFinalFile) gprint('Copy of final linkTable written to '+ linkTableFinalFile) gprint('Creating shapefiles with linework for links.') try: lu.write_link_maps(outlinkTableFile, step=5) except Exception: lu.write_link_maps(outlinkTableFile, step=5) # Create final linkmap files in output directory, and remove files from # scratch. lu.copy_final_link_maps(step=5) if not SAVENORMLCCS: lu.delete_dir(cfg.LCCBASEDIR) # Build statistics for corridor rasters arcpy.AddMessage('\nBuilding output statistics and pyramids ' 'for corridor raster') lu.build_stats(intRaster) if writeTruncRaster: arcpy.AddMessage('Building output statistics ' 'for truncated corridor raster') lu.build_stats(truncRaster) save_parameters() if cfg.OUTPUTFORMODELBUILDER: arcpy.CopyFeatures_management(cfg.COREFC, cfg.OUTPUTFORMODELBUILDER) # Return GEOPROCESSING specific errors except arcpy.ExecuteError: lu.dashline(1) gprint('****Failed in step 5. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except Exception: lu.dashline(1) gprint('****Failed in step 5. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME) return
def STEP3_calc_cwds(): """Calculates cost-weighted distances from each core area. Uses bounding circles around source and target cores to limit extent of cwd calculations and speed computation. """ try: lu.dashline(1) gprint('Running script ' + _SCRIPT_NAME) lu.dashline(0) # Super secret setting to re-start failed run. Enter 'RESTART' as the # Name of the pairwise distance table in step 2, and uncheck step 2. # We can eventually place this in a .ini file. rerun = False if cfg.S2EUCDISTFILE != None: if cfg.S2EUCDISTFILE.lower() == "restart": rerun = True # if cfg.TMAXCWDIST is None: # gprint('NOT using a maximum cost-weighted distance.') # else: # gprint('Max cost-weighted distance for CWD calcs set ' # 'to ' + str(cfg.TMAXCWDIST) + '\n') if (cfg.BUFFERDIST) is not None: gprint('Bounding circles plus a buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units will ' 'be used \n to limit extent of cost distance ' 'calculations.') elif cfg.TOOL <> cfg.TOOL_CC: gprint('NOT using bounding circles in cost distance ' 'calculations.') # set the analysis extent and cell size # So we don't extract rasters that go beyond extent of original raster if arcpy: arcpy.env.cellSize = cfg.RESRAST arcpy.env.extent="MINOF" else: gp.cellSize = gp.Describe(cfg.RESRAST).MeanCellHeight gp.Extent = "MINOF" gp.mask = cfg.RESRAST if arcpy: arcpy.env.overwriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR else: gp.OverwriteOutput = True gp.workspace = cfg.SCRATCHDIR gp.scratchWorkspace = cfg.ARCSCRATCHDIR # Load linkTable (created in previous script) linkTableFile = lu.get_prev_step_link_table(step=3) linkTable = lu.load_link_table(linkTableFile) lu.report_links(linkTable) # Identify cores to map from LinkTable coresToMap = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) numCoresToMap = len(coresToMap) if numCoresToMap < 3: # No need to check for intermediate cores, because there aren't any cfg.S3DROPLCCSic = False else: cfg.S3DROPLCCSic = cfg.S3DROPLCCS gprint('\nNumber of core areas to connect: ' + str(numCoresToMap)) if rerun: # If picking up a failed run, make sure needed files are there lu.dashline(1) gprint ('\n****** RESTART MODE ENABLED ******\n') gprint ('**** NOTE: This mode picks up step 3 where a\n' 'previous run left off due to a crash or user\n' 'abort. It assumes you are using the same input\n' 'data used in the terminated run.\n\n') lu.warn('IMPORTANT: Your LCP and stick feature classes\n' 'will LOSE LCPs that were already created, but\n' 'your final raster corridor map should be complete.\n') lu.dashline(0) lu.snooze(10) savedLinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") coreListFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") if not path.exists(savedLinkTableFile) or not path.exists( coreListFile): gprint('No partial results file found from previous ' 'stopped run. Starting run from beginning.\n') lu.dashline(0) rerun = False # If picking up a failed run, use old folders if not rerun: startIndex = 0 if cfg.TOOL <> cfg.TOOL_CC: lu.make_cwd_paths(max(coresToMap)) # Set up cwd directories # make a feature layer for input cores to select from gp.MakeFeatureLayer(cfg.COREFC, cfg.FCORES) # Drop links that are too long gprint('\nChecking for corridors that are too long to map.') DISABLE_LEAST_COST_NO_VAL = False linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, 0, cfg.MAXCOSTDIST, 0, DISABLE_LEAST_COST_NO_VAL) # ------------------------------------------------------------------ # Bounding boxes if (cfg.BUFFERDIST) is not None: # create bounding boxes around cores start_time = time.clock() # lu.dashline(1) gprint('Calculating bounding boxes for core areas.') extentBoxList = npy.zeros((0,5), dtype='float32') for x in range(len(coresToMap)): core = coresToMap[x] boxCoords = lu.get_extent_box_coords(core) extentBoxList = npy.append(extentBoxList, boxCoords, axis=0) gprint('\nDone calculating bounding boxes.') start_time = lu.elapsed_time(start_time) # lu.dashline() # Bounding circle code if cfg.BUFFERDIST is not None: # Make a set of circles encompassing core areas we'll be connecting start_time = time.clock() gprint('Calculating bounding circles around potential' ' corridors.') # x y corex corey radius- stores data for bounding circle centroids boundingCirclePointArray = npy.zeros((0,5), dtype='float32') circleList = npy.zeros((0,3), dtype='int32') numLinks = linkTable.shape[0] for x in range(0, numLinks): if ((linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_CORR) or (linkTable[x,cfg.LTB_LINKTYPE] == cfg.LT_KEEP)): # if it's a valid corridor link linkId = int(linkTable[x,cfg.LTB_LINKID]) # fixme- this code is clumsy- can trim down cores = npy.zeros((1,3), dtype='int32') cores[0,:] = npy.sort([0, linkTable[x,cfg.LTB_CORE1], linkTable[x,cfg.LTB_CORE2]]) corex = cores[0,1] corey = cores[0,2] cores[0,0] = linkId ################### foundFlag = False for y in range(0,len(circleList)): # clumsy if (circleList[y,1] == corex and circleList[y,2] == corey): foundFlag = True if not foundFlag: circlePointData = ( lu.get_bounding_circle_data(extentBoxList, corex, corey, cfg.BUFFERDIST)) boundingCirclePointArray = ( npy.append(boundingCirclePointArray, circlePointData, axis=0)) # keep track of which cores we draw bounding circles # around circleList = npy.append(circleList, cores, axis=0) gprint('\nCreating bounding circles using buffer ' 'analysis.') dir, BNDCIRCENS = path.split(cfg.BNDCIRCENS) lu.make_points(cfg.SCRATCHDIR, boundingCirclePointArray, BNDCIRCENS) lu.delete_data(cfg.BNDCIRS) gp.buffer_analysis(cfg.BNDCIRCENS, cfg.BNDCIRS, "radius") gp.deletefield (cfg.BNDCIRS, "BUFF_DIST") gprint('Successfully created bounding circles around ' 'potential corridors using \na buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.') start_time = lu.elapsed_time(start_time) gprint('Reducing global processing area using bounding ' 'circle plus buffer of ' + str(float(cfg.BUFFERDIST)) + ' map units.\n') extentBoxList = npy.zeros((0,5),dtype='float32') boxCoords = lu.get_extent_box_coords() extentBoxList = npy.append(extentBoxList,boxCoords,axis=0) extentBoxList[0,0] = 0 boundingCirclePointArray = npy.zeros((0,5),dtype='float32') circlePointData=lu.get_bounding_circle_data(extentBoxList, 0, 0, cfg.BUFFERDIST) dir, BNDCIRCEN = path.split(cfg.BNDCIRCEN) lu.make_points(cfg.SCRATCHDIR, circlePointData, BNDCIRCEN) lu.delete_data(cfg.BNDCIR) gp.buffer_analysis(cfg.BNDCIRCEN, cfg.BNDCIR, "radius") gprint('Extracting raster....') cfg.BOUNDRESIS = cfg.BOUNDRESIS + tif lu.delete_data(cfg.BOUNDRESIS) count = 0 statement = ( 'gp.ExtractByMask_sa(cfg.RESRAST, cfg.BNDCIR, cfg.BOUNDRESIS)') while True: try: exec statement randomerror() except: count,tryAgain = lu.retry_arc_error(count,statement) if not tryAgain: exec statement else: break gprint('\nReduced resistance raster extracted using ' 'bounding circle.') else: #if not using bounding circles, just go with resistance raster. cfg.BOUNDRESIS = cfg.RESRAST # --------------------------------------------------------------------- # Rasterize core areas to speed cost distance calcs # lu.dashline(1) gprint("Creating core area raster.") gp.SelectLayerByAttribute(cfg.FCORES, "CLEAR_SELECTION") if arcpy: arcpy.env.cellSize = cfg.BOUNDRESIS arcpy.env.extent = cfg.BOUNDRESIS else: gp.cellSize = gp.Describe(cfg.BOUNDRESIS).MeanCellHeight gp.extent = gp.Describe(cfg.BOUNDRESIS).extent if rerun: # saved linktable replaces the one now in memory linkTable = lu.load_link_table(savedLinkTableFile) coresToMapSaved = npy.loadtxt(coreListFile, dtype='Float64', comments='#', delimiter=',') startIndex = coresToMapSaved[0] # Index of core where we left off del coresToMapSaved gprint ('\n****** Re-starting run at core area number ' + str(int(coresToMap[startIndex]))+ ' ******\n') lu.dashline(0) if arcpy: arcpy.env.extent = "MINOF" else: gp.extent = "MINOF" #---------------------------------------------------------------------- # Loop through cores, do cwd calcs for each if cfg.TOOL == cfg.TOOL_CC: gprint("\nMapping least-cost paths.\n") else: gprint("\nStarting cost distance calculations.\n") lcpLoop = 0 failures = 0 x = startIndex endIndex = len(coresToMap) linkTableMod = linkTable.copy() while x < endIndex: startTime1 = time.clock() # Modification of linkTable in function was causing problems. so # make a copy: linkTablePassed = linkTableMod.copy() (linkTableReturned, failures, lcpLoop) = do_cwd_calcs(x, linkTablePassed, coresToMap, lcpLoop, failures) if failures == 0: # If iteration was successful, continue with next core linkTableMod = linkTableReturned sourceCore = int(coresToMap[x]) gprint('Done with all calculations for core ID #' + str(sourceCore) + '. ' + str(int(x + 1)) + ' of ' + str(endIndex) + ' cores have been processed.') start_time = lu.elapsed_time(startTime1) outlinkTableFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.write_link_table(linkTableMod, outlinkTableFile) # Increment loop counter x = x + 1 else: # If iteration failed, try again after a wait period delay_restart(failures) #---------------------------------------------------------------------- linkTable = linkTableMod # reinstate temporarily disabled links rows = npy.where(linkTable[:,cfg.LTB_LINKTYPE] > 1000) linkTable[rows,cfg.LTB_LINKTYPE] = (linkTable[rows,cfg.LTB_LINKTYPE] - 1000) # Drop links that are too long DISABLE_LEAST_COST_NO_VAL = True linkTable,numDroppedLinks = lu.drop_links(linkTable, cfg.MAXEUCDIST, cfg.MINEUCDIST, cfg.MAXCOSTDIST, cfg.MINCOSTDIST, DISABLE_LEAST_COST_NO_VAL) # Write link table file outlinkTableFile = lu.get_this_step_link_table(step=3) gprint('Updating ' + outlinkTableFile) lu.write_link_table(linkTable, outlinkTableFile) linkTableLogFile = path.join(cfg.LOGDIR, "linkTable_s3.csv") lu.write_link_table(linkTable, linkTableLogFile) start_time = time.clock() gprint('Creating shapefiles with linework for links...') try: lu.write_link_maps(outlinkTableFile, step=3) except: lu.write_link_maps(outlinkTableFile, step=3) start_time = lu.elapsed_time(start_time) gprint('\nIndividual cost-weighted distance layers written ' 'to "cwd" directory. \n') gprint(outlinkTableFile + '\n updated with cost-weighted distances between core areas.') #Clean up temporary files for restart code tempFile = path.join(cfg.DATAPASSDIR, "temp_cores_to_map.csv") lu.delete_file(tempFile) tempFile = path.join(cfg.DATAPASSDIR, "temp_linkTable_s3_partial.csv") lu.delete_file(tempFile) # Check if climate tool is calling linkage mapper if cfg.TOOL == cfg.TOOL_CC: coreList = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) for core in coreList: cwdRaster = lu.get_cwd_path(int(core)) back_rast = cwdRaster.replace("cwd_", "back_") lu.delete_data(back_rast) # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 3. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME) return
def lm_master(argv=None): """Main function for linkage mapper. Called by ArcMap with parameters or run from command line with parameters entered in script below. Calls functions in dedicated scripts for each of 5 processing steps. """ # Setup global variables if not cfg.lm_configured: # Causing problems with iterative scripting if argv is None: argv = sys.argv cfg.configure(cfg.TOOL_LM, argv) gp = cfg.gp try: gprint = lu.gprint # Move results from earlier versions to new directory structure lu.move_old_results() gp.OverwriteOutput = True gp.pyramid = "NONE" gp.rasterstatistics = "NONE" # Create output directories if they don't exist if gp.Exists(cfg.OUTPUTDIR): gp.RefreshCatalog(cfg.OUTPUTDIR) lu.create_dir(cfg.OUTPUTDIR) lu.create_dir(cfg.LOGDIR) lu.create_dir(cfg.MESSAGEDIR) lu.create_dir(cfg.DATAPASSDIR) # Create fresh scratch directory if not restarting in midst of step 3 # if cfg.S2EUCDISTFILE != None: # if cfg.S2EUCDISTFILE.lower() == "restart": pass # else: lu.delete_dir(cfg.SCRATCHDIR) lu.create_dir(cfg.SCRATCHDIR) lu.create_dir(cfg.ARCSCRATCHDIR) if cfg.TOOL == 'Linkage Mapper': cfg.logFilePath = lu.create_log_file(cfg.MESSAGEDIR, cfg.TOOL, cfg.PARAMS) lu.print_drive_warning() installD = gp.GetInstallInfo("desktop") gprint('\nLinkage Mapper Version ' + cfg.releaseNum) try: gprint('on ArcGIS ' + installD['ProductName'] + ' ' + installD['Version'] + ' Service Pack ' + installD['SPNumber']) except: pass if cfg.CONNECTFRAGS: # gwarn = gp.AddWarning lu.dashline(1) lu.warn( 'Custom mode: will run steps 1-2 ONLY to cluster core polygons within ' ) lu.warn( 'the maximum Euclidean corridor distance from one another ') lu.warn('into polygons with a single cluster_ID value.') lu.warn( 'Make sure you have set a Maximum Euclidean corridor distance.' ) lu.dashline(2) cfg.STEP3 = False cfg.STEP4 = False cfg.STEP5 = False if cfg.MAXEUCDIST == None: raise RuntimeError('Maximum Euclidean distance required ' 'for custom cluster mode.') # Set data frame spatial reference to coordinate system of input data # Problems arise in this script (core raster creation) and in S2 # (generate near table) if they differ. lu.set_dataframe_sr() # Check core ID field and project directory name. lu.check_cores(cfg.COREFC, cfg.COREFN) lu.check_project_dir() # Identify first step cleanup link tables from that point lu.dashline(1) if cfg.STEP1: gprint('Starting at step 1.') firststep = 1 elif cfg.STEP2: gprint('Starting at step 2.') firststep = 2 elif cfg.STEP3: gprint('Starting at step 3.') firststep = 3 linkTableFile = lu.get_prev_step_link_table(step=3) # Check exists elif cfg.STEP4: gprint('Starting at step 4.') firststep = 4 linkTableFile = lu.get_prev_step_link_table(step=4) # Check exists elif cfg.STEP5: gprint('Starting at step 5.') firststep = 5 linkTableFile = lu.get_prev_step_link_table(step=5) # Check exists lu.clean_up_link_tables(firststep) # Make a local grid copy of resistance raster for cwd runs- # will run faster than gdb. # Don't know if raster is in a gdb if entered from TOC lu.delete_data(cfg.RESRAST) gprint('\nMaking temporary copy of resistance raster for this run.') gp.OutputCoordinateSystem = gp.describe(cfg.COREFC).SpatialReference gp.Extent = gp.Describe(cfg.RESRAST_IN).Extent gp.SnapRaster = cfg.RESRAST_IN gp.cellSize = gp.Describe(cfg.RESRAST_IN).MeanCellHeight # import pdb; pdb.set_trace() try: gp.CopyRaster_management(cfg.RESRAST_IN, cfg.RESRAST) except: msg = ('ERROR: Could not make a copy of your resistance raster. ' + 'Try re-starting ArcMap to release the file lock.') lu.raise_error(msg) if (cfg.STEP1) or (cfg.STEP3): # Make core raster file gprint('\nMaking temporary raster of core file for this run.') lu.delete_data(cfg.CORERAS) gp.FeatureToRaster_conversion( cfg.COREFC, cfg.COREFN, cfg.CORERAS, gp.Describe(cfg.RESRAST).MeanCellHeight) # # gp.RasterToPolygon_conversion(cfg.CORERAS, cfg.COREFC, # "NO_SIMPLIFY") def delete_final_gdb(finalgdb): """Deletes final geodatabase""" if gp.Exists(finalgdb) and cfg.STEP5: try: lu.clean_out_workspace(finalgdb) except: lu.dashline(1) msg = ('ERROR: Could not remove contents of geodatabase ' + finalgdb + '. \nIs it open in ArcMap? You may ' 'need to re-start ArcMap to release the file lock.') lu.raise_error(msg) lu.delete_dir(finalgdb) # Delete final output geodatabase delete_final_gdb(cfg.OUTPUTGDB_OLD) delete_final_gdb(cfg.OUTPUTGDB) delete_final_gdb(cfg.EXTRAGDB) delete_final_gdb(cfg.LINKMAPGDB) # Run linkage mapper processing steps if cfg.STEP1: s1.STEP1_get_adjacencies() if cfg.STEP2: s2.STEP2_build_network() if cfg.STEP3: s3.STEP3_calc_cwds() if cfg.STEP4: s4.STEP4_refine_network() if cfg.STEP5: s5.STEP5_calc_lccs() lu.dashline() gprint('Results from this run can be found in your output ' 'directory:') gprint(cfg.OUTPUTDIR) # Clean up lu.delete_dir(cfg.SCRATCHDIR) # lu.delete_data(cfg.FCORES) gp.addmessage('\nDone with linkage mapping.\n') # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.exit_with_python_error(_SCRIPT_NAME) finally: lu.dashline() gprint('A record of run settings and messages can be found in your ' 'log directory:') gprint(cfg.MESSAGEDIR) lu.dashline(2) lu.close_log_file()
def setup_wrkspace(gisdbase, ccr_grassrc, geo_file): """Setup GRASS workspace""" lm_util.gprint("Creating GRASS workspace") gisbase = cc_env.gisbase location = "gcwd" mapset = "PERMANENT" os.environ['GISRC'] = ccr_grassrc os.environ['LD_LIBRARY_PATH'] = os.path.join(gisbase, "lib") os.environ['GRASS_SH'] = os.path.join(gisbase, "msys", "bin", "sh.exe") try: grass.create_location(gisdbase, location, filename=geo_file) except: lm_util.warn("GRASS ERROR. Try rebooting and restarting ArcGIS.") lm_util.warn("If that doesn't work you can try using ") lm_util.warn("the 'CC Run Script.py' python script in the ") lm_util.warn("demo directory where the Linkage Mapper toolbox") lm_util.warn("is installed instead of ArcGIS to call the tool") lm_util.warn("(see user guide).") raise Exception("GRASS ERROR: Cannot create workspace.") gsetup.init(gisbase, gisdbase, location, mapset) run_grass_cmd("g.gisenv", set="OVERWRITE=1") os.environ['GRASS_VERBOSE'] = "0" # Only errors and warnings are printed
def STEP8_calc_pinchpoints(): """ Maps pinch points in Linkage Mapper corridors using Circuitscape given CWD calculations from s3_calcCwds.py. """ try: lu.dashline(0) gprint('Running script ' + _SCRIPT_NAME) restartFlag = False if cfg.CWDCUTOFF < 0: cfg.CWDCUTOFF = cfg.CWDCUTOFF * -1 restartFlag = True # Restart code in progress CSPATH = lu.get_cs_path() outputGDB = path.join(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) arcpy.OverWriteOutput = True arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.pyramid = "NONE" arcpy.env.rasterstatistics = "NONE" # set the analysis extent and cell size to that of the resistance # surface arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST arcpy.snapraster = cfg.RESRAST resRaster = cfg.RESRAST arcpy.env.extent = "MINOF" minObject = arcpy.GetRasterProperties_management(resRaster, "MINIMUM") rasterMin = float(str(minObject.getOutput(0))) if rasterMin <= 0: msg = ( 'Error: resistance raster cannot have 0 or negative values.') lu.raise_error(msg) if cfg.DO_ADJACENTPAIRS: prevLcpShapefile = lu.get_lcp_shapefile(None, thisStep=8) if not arcpy.Exists(prevLcpShapefile): msg = ('Cannot find an LCP shapefile from step 5. Please ' 'rerun that step and any previous ones if necessary.') lu.raise_error(msg) # Remove lcp shapefile lcpShapefile = path.join(cfg.DATAPASSDIR, "lcpLines_s8.shp") lu.delete_data(lcpShapefile) inLinkTableFile = lu.get_prev_step_link_table(step=8) linkTable = lu.load_link_table(inLinkTableFile) numLinks = linkTable.shape[0] numCorridorLinks = lu.report_links(linkTable) if numCorridorLinks == 0: lu.dashline(1) msg = ('\nThere are no linkages. Bailing.') lu.raise_error(msg) if linkTable.shape[1] < 16: # If linktable has no entries from prior # centrality or pinchpint analyses extraCols = npy.zeros((numLinks, 6), dtype="float64") linkTable = linkTable[:, 0:10] linkTable = npy.append(linkTable, extraCols, axis=1) linkTable[:, cfg.LTB_LCPLEN] = -1 linkTable[:, cfg.LTB_CWDEUCR] = -1 linkTable[:, cfg.LTB_CWDPATHR] = -1 linkTable[:, cfg.LTB_EFFRESIST] = -1 linkTable[:, cfg.LTB_CWDTORR] = -1 linkTable[:, cfg.LTB_CURRENT] = -1 del extraCols # set up directories for circuit and circuit mosaic grids # Create output geodatabase if not arcpy.Exists(cfg.PINCHGDB): arcpy.CreateFileGDB_management(cfg.OUTPUTDIR, path.basename(cfg.PINCHGDB)) mosaicRaster = path.join(cfg.CIRCUITBASEDIR, "current_mos" + tif) coresToProcess = npy.unique(linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1]) maxCoreNum = max(coresToProcess) del coresToProcess lu.dashline(0) coreList = linkTable[:, cfg.LTB_CORE1:cfg.LTB_CORE2 + 1] coreList = npy.sort(coreList) #gprint('There are ' + str(len(npy.unique(coreList))) ' core areas.') INCIRCUITDIR = cfg.CIRCUITBASEDIR OUTCIRCUITDIR = path.join(cfg.CIRCUITBASEDIR, cfg.CIRCUITOUTPUTDIR_NM) CONFIGDIR = path.join(INCIRCUITDIR, cfg.CIRCUITCONFIGDIR_NM) # Cutoff value text to append to filenames cutoffText = str(cfg.CWDCUTOFF) if cutoffText[-6:] == '000000': cutoffText = cutoffText[0:-6] + 'm' elif cutoffText[-3:] == '000': cutoffText = cutoffText[0:-3] + 'k' if cfg.SQUARERESISTANCES: # Square resistance values squaredRaster = path.join(cfg.SCRATCHDIR, 'res_sqr') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR outRas = Raster(resRaster) * Raster(resRaster) outRas.save(squaredRaster) resRaster = squaredRaster if cfg.DO_ADJACENTPAIRS: linkLoop = 0 lu.dashline(1) gprint('Mapping pinch points in individual corridors \n' 'using Circuitscape.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(2) for x in range(0, numLinks): linkId = str(int(linkTable[x, cfg.LTB_LINKID])) if not (linkTable[x, cfg.LTB_LINKTYPE] > 0): continue linkLoop = linkLoop + 1 linkDir = path.join(cfg.SCRATCHDIR, 'link' + linkId) if restartFlag == True and path.exists(linkDir): gprint('continuing') continue restartFlag = False lu.create_dir(linkDir) start_time1 = time.clock() # source and target cores corex = int(coreList[x, 0]) corey = int(coreList[x, 1]) # Get cwd rasters for source and target cores cwdRaster1 = lu.get_cwd_path(corex) cwdRaster2 = lu.get_cwd_path(corey) lccNormRaster = path.join(linkDir, 'lcc_norm') arcpy.env.extent = "MINOF" link = lu.get_links_from_core_pairs(linkTable, corex, corey) lcDist = float(linkTable[link, cfg.LTB_CWDIST]) # Normalized lcc rasters are created by adding cwd rasters # and subtracting the least cost distance between them. outRas = Raster(cwdRaster1) + Raster(cwdRaster2) - lcDist outRas.save(lccNormRaster) #create raster mask resMaskRaster = path.join(linkDir, 'res_mask' + tif) #create raster mask outCon = arcpy.sa.Con( Raster(lccNormRaster) <= cfg.CWDCUTOFF, 1) outCon.save(resMaskRaster) # Convert to poly. Use as mask to clip resistance raster. resMaskPoly = path.join(linkDir, 'res_mask_poly.shp') arcpy.RasterToPolygon_conversion(resMaskRaster, resMaskPoly, "NO_SIMPLIFY") arcpy.env.extent = resMaskPoly # Includes 0 values in some cases with CP LI model if tif # so using ESRI Grid format resClipRasterMasked = path.join(linkDir, 'res_clip_m') # Extract masked resistance raster. # Needs to be float to get export to npy to work. outRas = arcpy.sa.ExtractByMask(resRaster, resMaskPoly) + 0.0 outRas.save(resClipRasterMasked) resNpyFN = 'resistances_link_' + linkId + '.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy( resClipRasterMasked, resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) lu.warn('Warning:') lu.warn('Circuitscape can only solve 2-3 million nodes') lu.warn( 'per gigabyte of available RAM. \nTotal physical RAM' ' on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~' + str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(2) corePairRaster = path.join(linkDir, 'core_pairs' + tif) arcpy.env.extent = resClipRasterMasked # Next result needs to be floating pt for numpy export outCon = arcpy.sa.Con( Raster(cwdRaster1) == 0, corex, arcpy.sa.Con(Raster(cwdRaster2) == 0, corey + 0.0)) outCon.save(corePairRaster) coreNpyFN = 'cores_link_' + linkId + '.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy( corePairRaster, coreNpyFile) arcpy.env.extent = "MINOF" # Set circuitscape options and call options = lu.setCircuitscapeOptions() if cfg.WRITE_VOLT_MAPS == True: options['write_volt_maps'] = True options['habitat_file'] = resNpyFile # if int(linkId) > 2: # options['habitat_file'] = 'c:\\test.dummy' options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero'] = True outputFN = 'Circuitscape_link' + linkId + '.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) if numElements > 250000: options['print_timings'] = True configFN = 'pinchpoint_config' + linkId + '.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('Processing link ID #' + str(linkId) + '. Resistance map' ' has ' + str(int(numResistanceNodes)) + ' nodes.') memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): print_failure(numResistanceNodes, memFlag, 10) numElements, numNodes = export_ras_to_npy( resClipRasterMasked, resNpyFile) memFlag = call_circuitscape(CSPATH, outConfigFile) currentFN = ('Circuitscape_link' + linkId + '_cum_curmap.npy') currentMap = path.join(OUTCIRCUITDIR, currentFN) if not arcpy.Exists(currentMap): msg = ( '\nCircuitscape failed. See error information above.') arcpy.AddError(msg) lu.write_log(msg) exit(1) # Either set core areas to nodata in current map or # divide each by its radius currentRaster = path.join(linkDir, "current" + tif) import_npy_to_ras(currentMap, corePairRaster, currentRaster) if cfg.WRITE_VOLT_MAPS == True: voltFN = ('Circuitscape_link' + linkId + '_voltmap_' + str(corex) + '_' + str(corey) + '.npy') voltMap = path.join(OUTCIRCUITDIR, voltFN) voltRaster = path.join( outputGDB, cfg.PREFIX + "_voltMap_" + str(corex) + '_' + str(corey)) import_npy_to_ras(voltMap, corePairRaster, voltRaster) gprint('Building output statistics and pyramids ' 'for voltage raster\n') lu.build_stats(voltRaster) arcpy.env.extent = currentRaster if SETCORESTONULL: # Set core areas to NoData in current map for color ramping currentRaster2 = currentRaster + '2' + tif outCon = arcpy.sa.Con( arcpy.sa.IsNull(Raster(corePairRaster)), Raster(currentRaster)) outCon.save(currentRaster2) currentRaster = currentRaster2 arcpy.env.extent = "MAXOF" if linkLoop == 1: lu.delete_data(mosaicRaster) @retry(10) def copyRas2(): arcpy.CopyRaster_management(currentRaster, mosaicRaster) copyRas2() else: @retry(10) def mosaicRas(): arcpy.Mosaic_management(currentRaster, mosaicRaster, "MAXIMUM", "MATCH") mosaicRas() resistancesFN = ('Circuitscape_link' + linkId + '_resistances_3columns.out') resistancesFile = path.join(OUTCIRCUITDIR, resistancesFN) resistances = npy.loadtxt(resistancesFile, dtype='Float64', comments='#') resistance = float(str(arcpy.env.cellSize)) * resistances[2] linkTable[link, cfg.LTB_EFFRESIST] = resistance # Ratio if not cfg.SQUARERESISTANCES: linkTable[link, cfg.LTB_CWDTORR] = ( linkTable[link, cfg.LTB_CWDIST] / linkTable[link, cfg.LTB_EFFRESIST]) # Clean up if cfg.SAVE_TEMP_CIRCUIT_FILES == False: lu.delete_file(coreNpyFile) coreNpyBase, extension = path.splitext(coreNpyFile) lu.delete_data(coreNpyBase + '.hdr') lu.delete_file(resNpyFile) resNpyBase, extension = path.splitext(resNpyFile) lu.delete_data(resNpyBase + '.hdr') lu.delete_file(currentMap) curMapBase, extension = path.splitext(currentMap) lu.delete_data(curMapBase + '.hdr') lu.delete_data(currentRaster) lu.clean_out_workspace(linkDir) lu.delete_dir(linkDir) gprint('Finished with link ID #' + str(linkId) + '. ' + str(linkLoop) + ' out of ' + str(numCorridorLinks) + ' links have been processed.') start_time1 = lu.elapsed_time(start_time1) outputRaster = path.join( outputGDB, cfg.PREFIX + "_current_adjacentPairs_" + cutoffText) lu.delete_data(outputRaster) @retry(10) def copyRas(): arcpy.CopyRaster_management(mosaicRaster, outputRaster) copyRas() gprint('Building output statistics and pyramids ' 'for corridor pinch point raster\n') lu.build_stats(outputRaster) finalLinkTable = lu.update_lcp_shapefile(linkTable, lastStep=5, thisStep=8) linkTableFile = path.join(cfg.DATAPASSDIR, "linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFile, inLinkTableFile) linkTableFinalFile = path.join( cfg.OUTPUTDIR, cfg.PREFIX + "_linkTable_s5_plus.csv") lu.write_link_table(finalLinkTable, linkTableFinalFile, inLinkTableFile) gprint('Copy of linkTable written to ' + linkTableFinalFile) #fixme: update sticks? gprint('Creating shapefiles with linework for links.') lu.write_link_maps(linkTableFinalFile, step=8) # Copy final link maps to gdb. lu.copy_final_link_maps(step=8) lu.delete_data(mosaicRaster) if not cfg.DO_ALLPAIRS: # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) return lu.dashline(1) gprint('Mapping global pinch points among all\n' 'core area pairs using Circuitscape.') if cfg.ALL_PAIR_SCENARIO == 'pairwise': gprint('Circuitscape will be run in PAIRWISE mode.') else: gprint('Circuitscape will be run in ALL-TO-ONE mode.') arcpy.env.workspace = cfg.SCRATCHDIR arcpy.env.scratchWorkspace = cfg.ARCSCRATCHDIR arcpy.env.extent = cfg.RESRAST arcpy.env.cellSize = cfg.RESRAST S8CORE_RAS = "s8core_ras" s8CoreRasPath = path.join(cfg.SCRATCHDIR, S8CORE_RAS) arcpy.FeatureToRaster_conversion(cfg.COREFC, cfg.COREFN, s8CoreRasPath, arcpy.env.cellSize) binaryCoreRaster = path.join(cfg.SCRATCHDIR, "core_ras_bin") # The following commands cause file lock problems on save. using gp # instead. # outCon = arcpy.sa.Con(S8CORE_RAS, 1, "#", "VALUE > 0") # outCon.save(binaryCoreRaster) # gp.Con_sa(s8CoreRasPath, 1, binaryCoreRaster, "#", "VALUE > 0") outCon = arcpy.sa.Con(Raster(s8CoreRasPath) > 0, 1) outCon.save(binaryCoreRaster) s5corridorRas = path.join(cfg.OUTPUTGDB, cfg.PREFIX + "_corridors") if not arcpy.Exists(s5corridorRas): s5corridorRas = path.join(cfg.OUTPUTGDB, cfg.PREFIX + "_lcc_mosaic_int") outCon = arcpy.sa.Con( Raster(s5corridorRas) <= cfg.CWDCUTOFF, Raster(resRaster), arcpy.sa.Con(Raster(binaryCoreRaster) > 0, Raster(resRaster))) resRasClipPath = path.join(cfg.SCRATCHDIR, 'res_ras_clip') outCon.save(resRasClipPath) arcpy.env.cellSize = resRasClipPath arcpy.env.extent = resRasClipPath s8CoreRasClipped = s8CoreRasPath + '_c' # Produce core raster with same extent as clipped resistance raster # added to ensure correct data type- nodata values were positive for # cores otherwise outCon = arcpy.sa.Con(arcpy.sa.IsNull(Raster(s8CoreRasPath)), -9999, Raster(s8CoreRasPath)) outCon.save(s8CoreRasClipped) resNpyFN = 'resistances.npy' resNpyFile = path.join(INCIRCUITDIR, resNpyFN) numElements, numResistanceNodes = export_ras_to_npy( resRasClipPath, resNpyFile) totMem, availMem = lu.get_mem() # gprint('Total memory: str(totMem)) if numResistanceNodes / availMem > 2000000: lu.dashline(1) lu.warn('Warning:') lu.warn('Circuitscape can only solve 2-3 million nodes') lu.warn('per gigabyte of available RAM. \nTotal physical RAM ' 'on your machine is ~' + str(totMem) + ' GB. \nAvailable memory is ~' + str(availMem) + ' GB. \nYour resistance raster has ' + str(numResistanceNodes) + ' nodes.') lu.dashline(0) coreNpyFN = 'cores.npy' coreNpyFile = path.join(INCIRCUITDIR, coreNpyFN) numElements, numNodes = export_ras_to_npy(s8CoreRasClipped, coreNpyFile) arcpy.env.extent = "MINOF" options = lu.setCircuitscapeOptions() options['scenario'] = cfg.ALL_PAIR_SCENARIO options['habitat_file'] = resNpyFile options['point_file'] = coreNpyFile options['set_focal_node_currents_to_zero'] = True outputFN = 'Circuitscape.out' options['output_file'] = path.join(OUTCIRCUITDIR, outputFN) options['print_timings'] = True configFN = 'pinchpoint_allpair_config.ini' outConfigFile = path.join(CONFIGDIR, configFN) lu.writeCircuitscapeConfigFile(outConfigFile, options) gprint('\nResistance map has ' + str(int(numResistanceNodes)) + ' nodes.') lu.dashline(1) gprint('If you try to cancel your run and the Arc dialog hangs, ') gprint('you can kill Circuitscape by opening Windows Task Manager') gprint('and ending the cs_run.exe process.') lu.dashline(0) call_circuitscape(CSPATH, outConfigFile) # test = subprocess.call([CSPATH, outConfigFile], # creationflags = subprocess.CREATE_NEW_CONSOLE) if options['scenario'] == 'pairwise': rasterSuffix = "_current_allPairs_" + cutoffText else: rasterSuffix = "_current_allToOne_" + cutoffText currentFN = 'Circuitscape_cum_curmap.npy' currentMap = path.join(OUTCIRCUITDIR, currentFN) outputRaster = path.join(outputGDB, cfg.PREFIX + rasterSuffix) currentRaster = path.join(cfg.SCRATCHDIR, "current") try: import_npy_to_ras(currentMap, resRasClipPath, outputRaster) except: lu.dashline(1) msg = ('ERROR: Circuitscape failed. \n' 'Note: Circuitscape can only solve 2-3 million nodes' '\nper gigabyte of available RAM. The resistance ' '\nraster for the last corridor had ' + str(numResistanceNodes) + ' nodes.\n\nResistance ' 'raster values that vary by >6 orders of \nmagnitude' ' can also cause failures, as can a mismatch in ' '\ncore area and resistance raster extents.') arcpy.AddError(msg) lu.write_log(msg) exit(1) #set core areas to nodata if SETCORESTONULL: # Set core areas to NoData in current map for color ramping outputRasterND = outputRaster + '_noDataCores' outCon = arcpy.sa.SetNull( Raster(s8CoreRasClipped) > 0, Raster(outputRaster)) outCon.save(outputRasterND) gprint('\nBuilding output statistics and pyramids ' 'for centrality raster.') lu.build_stats(outputRaster) lu.build_stats(outputRasterND) # Clean up temporary files if not cfg.SAVECURRENTMAPS: lu.delete_dir(OUTCIRCUITDIR) # Return GEOPROCESSING specific errors except arcpy.ExecuteError: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.dashline(1) gprint('****Failed in step 8. Details follow.****') lu.exit_with_python_error(_SCRIPT_NAME)
def lm_master(argv=None): """Main function for linkage mapper. Called by ArcMap with parameters or run from command line with parameters entered in script below. Calls functions in dedicated scripts for each of 5 processing steps. """ # Setup global variables if cfg.TOOL != cfg.TOOL_CC: if argv is None: argv = sys.argv cfg.configure(cfg.TOOL_LM, argv) gprint = lu.gprint try: # Move results from earlier versions to new directory structure lu.move_old_results() arcpy.env.pyramid = "NONE" arcpy.env.rasterStatistics = "NONE" # Create output directories if they don't exist lu.create_dir(cfg.OUTPUTDIR) lu.create_dir(cfg.LOGDIR) lu.create_dir(cfg.MESSAGEDIR) lu.create_dir(cfg.DATAPASSDIR) if cfg.TOOL != cfg.TOOL_CC: lu.delete_dir(cfg.SCRATCHDIR) lu.create_dir(cfg.SCRATCHDIR) lu.create_dir(cfg.ARCSCRATCHDIR) if cfg.TOOL == cfg.TOOL_LM: cfg.logFilePath = lu.create_log_file(cfg.PARAM_NAMES, argv) lu.write_custom_to_log(cfg.LMCUSTSETTINGS) lu.log_metadata(cfg.COREFC, [cfg.RESRAST_IN]) lu.print_drive_warning() if cfg.CONNECTFRAGS: lu.dashline(1) lu.warn('Custom mode: will run steps 1-2 ONLY to cluster core polygons within ') lu.warn('the maximum Euclidean corridor distance from one another ') lu.warn('into polygons with a single cluster_ID value.') lu.warn('Make sure you have set a Maximum Euclidean corridor distance.') lu.dashline(2) cfg.STEP3 = False cfg.STEP4 = False cfg.STEP5 = False if cfg.MAXEUCDIST == None: raise RuntimeError('Maximum Euclidean distance required ' 'for custom cluster mode.') # Set data frame spatial reference to coordinate system of input data # Problems arise in this script (core raster creation) and in S2 # (generate near table) if they differ. lu.set_dataframe_sr() # Check core ID field and project directory name. lu.check_cores(cfg.COREFC, cfg.COREFN) lu.check_project_dir() # Identify first step cleanup link tables from that point lu.dashline(1) if cfg.STEP1: gprint('Starting at step 1.') firststep = 1 elif cfg.STEP2: gprint('Starting at step 2.') firststep = 2 elif cfg.STEP3: gprint('Starting at step 3.') firststep = 3 linkTableFile = lu.get_prev_step_link_table(step=3) # Check exists elif cfg.STEP4: gprint('Starting at step 4.') firststep = 4 linkTableFile = lu.get_prev_step_link_table(step=4) # Check exists elif cfg.STEP5: gprint('Starting at step 5.') firststep = 5 linkTableFile = lu.get_prev_step_link_table(step=5) # Check exists lu.clean_up_link_tables(firststep) # Make a local grid copy of resistance raster for cwd runs- # will run faster than gdb. # Don't know if raster is in a gdb if entered from TOC lu.delete_data(cfg.RESRAST) gprint('\nMaking temporary copy of resistance raster for this run.') arcpy.env.outputCoordinateSystem = arcpy.Describe(cfg.COREFC).SpatialReference arcpy.env.extent = arcpy.Describe(cfg.RESRAST_IN).Extent arcpy.env.snapRaster = cfg.RESRAST_IN arcpy.env.cellSize = arcpy.Describe(cfg.RESRAST_IN).MeanCellHeight try: arcpy.CopyRaster_management(cfg.RESRAST_IN, cfg.RESRAST) except Exception: msg = ('ERROR: Could not make a copy of your resistance raster. ' + 'Try re-starting ArcMap to release the file lock.') lu.raise_error(msg) if (cfg.STEP1) or (cfg.STEP3): # Make core raster file gprint('\nMaking temporary raster of core file for this run.') lu.delete_data(cfg.CORERAS) arcpy.FeatureToRaster_conversion(cfg.COREFC, cfg.COREFN, cfg.CORERAS, arcpy.Describe(cfg.RESRAST).MeanCellHeight) def delete_final_gdb(finalgdb): """Deletes final geodatabase""" if arcpy.Exists(finalgdb) and cfg.STEP5: try: lu.clean_out_workspace(finalgdb) except Exception: lu.dashline(1) msg = ('ERROR: Could not remove contents of geodatabase ' + finalgdb + '. \nIs it open in ArcMap? You may ' 'need to re-start ArcMap to release the file lock.') lu.raise_error(msg) lu.delete_dir(finalgdb) # Delete final output geodatabase delete_final_gdb(cfg.OUTPUTGDB_OLD) delete_final_gdb(cfg.OUTPUTGDB) delete_final_gdb(cfg.EXTRAGDB) delete_final_gdb(cfg.LINKMAPGDB) # Run linkage mapper processing steps if cfg.STEP1: s1.STEP1_get_adjacencies() if cfg.STEP2: s2.STEP2_build_network() if cfg.STEP3: s3.STEP3_calc_cwds() if cfg.STEP4: s4.STEP4_refine_network() if cfg.STEP5: s5.STEP5_calc_lccs() lu.dashline() gprint('Results from this run can be found in your output ' 'directory:') gprint(cfg.OUTPUTDIR) # Clean up lu.delete_dir(cfg.SCRATCHDIR) arcpy.AddMessage('\nDone with linkage mapping.\n') # Return GEOPROCESSING specific errors except arcpy.ExecuteError: lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except Exception: lu.exit_with_python_error(_SCRIPT_NAME) finally: lu.dashline() gprint('A record of run settings and messages can be found in your ' 'log directory:') gprint(cfg.MESSAGEDIR) lu.dashline(2) lu.close_log_file()
def circuitscape_master(argv=None): """ """ gprint = lu.gprint # gwarn = arcpy.AddWarning if argv is None: argv = sys.argv cfg.configure(cfg.TOOL_CS, argv) gp = cfg.gp try: lu.create_dir(cfg.LOGDIR) lu.create_dir(cfg.MESSAGEDIR) cfg.logFilePath = lu.create_log_file(cfg.MESSAGEDIR, cfg.TOOL, cfg.PARAMS) CSPATH = lu.get_cs_path() if CSPATH == None: msg = ('Cannot find an installation of Circuitscape 3.5.5' '\nor greater in your Program Files directory.') arcpy.AddError(msg) lu.write_log(msg) exit(1) try: csDir, fn = path.split(CSPATH) if 'flush' not in open(path.join(csDir, 'cs_compute.py')).read(): lu.warn('\n---------------------------------------------') lu.warn('Your version of Circuitscape is out of date. ') lu.warn('---------------------------------------------\n') lu.warn('Please get the latest from www.circuitscape.org.') lu.warn('The new version interacts more smoothly with ArcMap.') gprint('Proceeding...\n') except: pass lu.print_drive_warning() # Check core ID field. lu.check_cores(cfg.COREFC, cfg.COREFN) gp.OutputCoordinateSystem = gp.describe(cfg.COREFC).SpatialReference # Set data frame spatial reference to coordinate system of input data lu.set_dataframe_sr() gp.pyramid = "NONE" gp.rasterstatistics = "NONE" # Move adj and cwd results from earlier versions to datapass directory lu.move_old_results() if cfg.CWDCUTOFF > 0: lu.delete_dir(cfg.SCRATCHDIR) # restart code- in progress if cfg.CWDCUTOFF < 0: cfg.CWDCUTOFF = cfg.CWDCUTOFF * -1 if not cfg.DOPINCH and not cfg.DOCENTRALITY: msg = ('ERROR: Please choose at least one option: pinch point or\n' 'network centrality analysis.') lu.raise_error(msg) lu.create_dir(cfg.SCRATCHDIR) lu.create_dir(cfg.ARCSCRATCHDIR) if cfg.DO_ALLPAIRS: # Fixme: move raster path to config S5CORRIDORRAS = path.join(cfg.OUTPUTGDB, cfg.PREFIX + "_corridors") if not gp.Exists(S5CORRIDORRAS): S5CORRIDORRAS = path.join(cfg.OUTPUTGDB, cfg.PREFIX + "_lcc_mosaic_int") if not gp.Exists(S5CORRIDORRAS): msg = ('ERROR: Corridor raster created in step 5 is required' '\nfor all-pair analyses, but was not found.') lu.raise_error(msg) if cfg.DOPINCH: if cfg.CWDCUTOFF == '#' or cfg.CWDCUTOFF == 0: msg = ('ERROR: CWD cutoff distance is required for pinch point' ' analyses.') lu.raise_error(msg) # Make a local grid copy of resistance raster- # will run faster than gdb. lu.delete_data(cfg.RESRAST) if not gp.Exists(cfg.RESRAST_IN): msg = ('ERROR: Resistance raster is required for pinch point' ' analyses, but was not found.') lu.raise_error(msg) desc = arcpy.Describe(cfg.RESRAST_IN) if hasattr(desc, "catalogPath"): cfg.RESRAST_IN = arcpy.Describe(cfg.RESRAST_IN).catalogPath # arcpy.env.extent = cfg.RESRAST_IN # arcpy.env.snapRaster = cfg.RESRAST_IN gprint('\nMaking local copy of resistance raster.') try: gp.CopyRaster_management(cfg.RESRAST_IN, cfg.RESRAST) except: msg = ( 'ERROR: Could not make a copy of your resistance raster. ' + 'Try re-starting ArcMap to release the file lock.') lu.raise_error(msg) arcpy.env.extent = cfg.RESRAST arcpy.env.snapRaster = cfg.RESRAST if cfg.DOCENTRALITY: gprint("Creating output folder: " + cfg.CENTRALITYBASEDIR) if path.exists(cfg.CENTRALITYBASEDIR): shutil.rmtree(cfg.CENTRALITYBASEDIR) lu.create_dir(cfg.CENTRALITYBASEDIR) gp.CreateFolder_management(cfg.CENTRALITYBASEDIR, cfg.CIRCUITOUTPUTDIR_NM) gp.CreateFolder_management(cfg.CENTRALITYBASEDIR, cfg.CIRCUITCONFIGDIR_NM) lu.clean_out_workspace(cfg.CORECENTRALITYGDB) s7.STEP7_calc_centrality() if not cfg.SAVECENTRALITYDIR: lu.delete_dir(cfg.CENTRALITYBASEDIR) if cfg.DOPINCH: if cfg.CWDCUTOFF > 0: # Negative values mean we're restarting gprint("Creating output folder: " + cfg.CIRCUITBASEDIR) lu.delete_dir(cfg.CIRCUITBASEDIR) lu.create_dir(cfg.CIRCUITBASEDIR) gp.CreateFolder_management(cfg.CIRCUITBASEDIR, cfg.CIRCUITOUTPUTDIR_NM) gp.CreateFolder_management(cfg.CIRCUITBASEDIR, cfg.CIRCUITCONFIGDIR_NM) s8.STEP8_calc_pinchpoints() if not cfg.SAVE_TEMP_CIRCUIT_FILES: lu.delete_dir(cfg.SCRATCHDIR) if not cfg.SAVECIRCUITDIR: lu.delete_dir(cfg.CIRCUITBASEDIR) gprint('\nDONE!\n') # Return GEOPROCESSING specific errors except arcgisscripting.ExecuteError: lu.exit_with_geoproc_error(_SCRIPT_NAME) # Return any PYTHON or system specific errors except: lu.exit_with_python_error(_SCRIPT_NAME)