def update_cache(map_service_path, scratch_workspace, gfw_env): logging.debug('Updating tiles for cached map service {0}'.format(map_service_path)) # Start the service on localhost-- should be shut off when not in use manage_service('dev', map_service_path, 'start') source_mxd, local_cache_dir = find_src_mxd_and_cache_dir(map_service_path) logging.debug("Found source MXD: {0}".format(source_mxd)) output_dir = util.create_temp_dir(scratch_workspace) # Zoom levels 0 - 6 scale_aoi = "591657527.591555;295828763.795777;147914381.897889;73957190.948944;" \ "36978595.474472;18489297.737236;9244648.868618" min_scale = scale_aoi.split(';')[0] max_scale = scale_aoi.split(';')[-1] logging.debug("Generating tiles . . . ") cache_dir_name = 'cache' arcpy.ManageTileCache_management(output_dir, "RECREATE_ALL_TILES", cache_dir_name, source_mxd, "ARCGISONLINE_SCHEME", "", scale_aoi, "", "", min_scale, max_scale) logging.debug("Copying to local and production cache directories") src_cache_dir = os.path.join(output_dir, cache_dir_name, 'Layers') if gfw_env == 'prod': push_to_production(src_cache_dir, local_cache_dir, map_service_path) else: logging.debug("Nothing pushed to prod dir; just testing cache generation process") logging.debug('Cached tiles are here: {0}'.format(local_cache_dir)) # Stop the map service-- no need for it to be serving on the DM machine manage_service('dev', map_service_path, 'stop')
def vector_to_raster(self, input_fc): if self.vector_to_raster_output: temp_dir = util.create_temp_dir(self.scratch_workspace) arcpy.CreateFileGDB_management(temp_dir, 'temp.gdb') # Get spatial reference of output sr = arcpy.Describe(self.vector_to_raster_output).spatialReference logging.debug('Starting to project input vector FC to the spatial reference of the output raster') out_projected_fc = os.path.join(temp_dir, 'temp.gdb', 'src_prj_to_ras_sr') arcpy.Project_management(input_fc, out_projected_fc, sr) util.add_field_and_calculate(out_projected_fc, 'ras_val', 'SHORT', '', 1, self.gfw_env) # Get cell size of output cell_size = int(arcpy.GetRasterProperties_management(self.vector_to_raster_output, 'CELLSIZEX').getOutput(0)) arcpy.env.pyramid = "NONE" arcpy.env.snapRaster = self.vector_to_raster_output logging.debug('Rasterizing and outputting as tif') out_raster = os.path.join(temp_dir, 'out.tif') arcpy.PolygonToRaster_conversion(out_projected_fc, 'ras_val', out_raster, "CELL_CENTER", "", cell_size) # Stop service that has a lock on the raster service = r'image_services/analysis' # arcgis_server.set_service_status(service, 'stop') logging.debug('Sleeping for 10 seconds to let the lock files resolve themselves') time.sleep(10) logging.debug('Copying raster {0} to output {1}'.format(out_raster, self.vector_to_raster_output)) arcpy.Delete_management(self.vector_to_raster_output) # Move all related tif files to final destination # Much faster than using CopyRaster_management-- just need to physically move the files src_dir = os.path.dirname(out_raster) src_file_name = os.path.splitext(os.path.basename(out_raster))[0] out_dir = os.path.dirname(self.vector_to_raster_output) out_file_name = os.path.splitext(os.path.basename(self.vector_to_raster_output))[0] for extension in ['.tif', '.tfw', '.tif.aux.xml', '.tif.vat.cpg', '.tif.vat.dbf', '.tif.xml']: src_file = os.path.join(src_dir, src_file_name + extension) out_file = os.path.join(out_dir, out_file_name + extension) try: shutil.move(src_file, out_file) except IOError: print 'No such file {0}'.format(src_file) # Restart the service after we're finished # arcgis_server.set_service_status(service, 'start') else: pass
def update_cache(map_service_path, scratch_workspace): logging.debug('Updating tiles for cached map service {0}'.format(map_service_path)) # Start the service on localhost-- should be shut off when not in use manage_service('dev', map_service_path, 'start') source_mxd, local_cache_dir = find_src_mxd_and_cache_dir(map_service_path) logging.debug("Found source MXD: {0}".format(source_mxd)) output_dir = util.create_temp_dir(scratch_workspace) # Zoom levels 0 - 6 scale_aoi = "591657527.591555;295828763.795777;147914381.897889;73957190.948944;" \ "36978595.474472;18489297.737236;9244648.868618" min_scale = scale_aoi.split(';')[0] max_scale = scale_aoi.split(';')[-1] logging.debug("Generating tiles . . . ") cache_dir_name = 'cache' arcpy.ManageTileCache_management(output_dir, "RECREATE_ALL_TILES", cache_dir_name, source_mxd, "ARCGISONLINE_SCHEME", "", scale_aoi, "", "", min_scale, max_scale) logging.debug("Copying to local and production cache directories") src_cache_dir = os.path.join(output_dir, cache_dir_name, 'Layers') push_to_production(src_cache_dir, local_cache_dir, map_service_path) # Stop the map service-- no need for it to be serving on the DM machine manage_service('dev', map_service_path, 'stop')
def main(): parser = argparse.ArgumentParser(description='QC loss and extent stats') parser.add_argument('--number-of-tiles', '-n', help='number of tiles to run QC', default=10, type=int, required=True) parser.add_argument('--grid-resolution', '-g', help='grid resolution of source', type=int, default=10, choices=(10, 0.25)) parser.add_argument('--s3-poly-dir', '-s', help='input poly directory for intersected TSVs', required=True) parser.add_argument('--thread-count', '-c', type=int, default=mp.cpu_count()) parser.add_argument('--test', dest='test', action='store_true') args = parser.parse_args() util.start_logging() # load gadm28 - required to determine which gadm28 boundaries are completely within a 10x10 tile load_gadm28.load('s3://gfw2-data/alerts-tsv/gis_source/adm2_final.zip') if args.test: args.number_of_tiles = 1 tile_list = s3_list_tiles.pull_random(args.s3_poly_dir, args.number_of_tiles) print tile_list temp_dir = util.create_temp_dir() process_list = [] for tile in tile_list: process_list.append((tile, args.s3_poly_dir, temp_dir)) util.exec_multiprocess(qc_output_tile, process_list, args.test, args.thread_count) qc.check_results()
def update_cache(map_service_path, scratch_workspace, gfw_env): logging.debug("Updating tiles for cached map service {0}".format(map_service_path)) # Start the service on localhost-- should be shut off when not in use manage_service("dev", map_service_path, "start") source_mxd, local_cache_dir = find_src_mxd_and_cache_dir(map_service_path) logging.debug("Found source MXD: {0}".format(source_mxd)) output_dir = util.create_temp_dir(scratch_workspace) # Zoom levels 0 - 6 scale_aoi = ( "591657527.591555;295828763.795777;147914381.897889;73957190.948944;" "36978595.474472;18489297.737236;9244648.868618" ) min_scale = scale_aoi.split(";")[0] max_scale = scale_aoi.split(";")[-1] logging.debug("Generating tiles . . . ") cache_dir_name = "cache" arcpy.ManageTileCache_management( output_dir, "RECREATE_ALL_TILES", cache_dir_name, source_mxd, "ARCGISONLINE_SCHEME", "", scale_aoi, "", "", min_scale, max_scale, ) logging.debug("Copying to local and production cache directories") src_cache_dir = os.path.join(output_dir, cache_dir_name, "Layers") if gfw_env == "PROD": push_to_production(src_cache_dir, local_cache_dir, map_service_path) else: logging.debug("Nothing pushed to PROD dir; just testing cache generation process") logging.debug("Cached tiles are here: {0}".format(local_cache_dir)) # Stop the map service-- no need for it to be serving on the DM machine manage_service("dev", map_service_path, "stop")
def vector_to_raster(self, input_fc): if self.vector_to_raster_output: temp_dir = util.create_temp_dir(self.scratch_workspace) arcpy.CreateFileGDB_management(temp_dir, 'temp.gdb') # Get spatial reference of output sr = arcpy.Describe(self.vector_to_raster_output).spatialReference logging.debug('Starting to project input vector FC to the spatial reference of the output raster') out_projected_fc = os.path.join(temp_dir, 'temp.gdb', 'src_prj_to_ras_sr') arcpy.Project_management(input_fc, out_projected_fc, sr) util.add_field_and_calculate(out_projected_fc, 'ras_val', 'SHORT', '', 1, self.gfw_env) # Get cell size of output cell_size = int(arcpy.GetRasterProperties_management(self.vector_to_raster_output, 'CELLSIZEX').getOutput(0)) arcpy.env.pyramid = "NONE" arcpy.env.snapRaster = self.vector_to_raster_output logging.debug('Rasterizing and outputting as tif') out_raster = os.path.join(temp_dir, 'out.tif') arcpy.PolygonToRaster_conversion(out_projected_fc, 'ras_val', out_raster, "CELL_CENTER", "", cell_size) logging.debug('Copying raster {0} to output {1}'.format(out_raster, self.vector_to_raster_output)) arcpy.Delete_management(self.vector_to_raster_output) # Move all related tif files to final destination # Much faster than using CopyRaster_management-- just need to physically move the files src_dir = os.path.dirname(out_raster) src_file_name = os.path.splitext(os.path.basename(out_raster))[0] out_dir = os.path.dirname(self.vector_to_raster_output) out_file_name = os.path.splitext(os.path.basename(self.vector_to_raster_output))[0] for extension in ['.tif', '.tfw', '.tif.aux.xml', '.tif.vat.cpg', '.tif.vat.dbf', '.tif.xml']: src_file = os.path.join(src_dir, src_file_name + extension) out_file = os.path.join(out_dir, out_file_name + extension) shutil.move(src_file, out_file) else: pass
def vector_to_raster(self, input_fc): if self.vector_to_raster_output: temp_dir = util.create_temp_dir(self.scratch_workspace) arcpy.CreateFileGDB_management(temp_dir, 'temp.gdb') # Get spatial reference of output sr = arcpy.Describe(self.vector_to_raster_output).spatialReference logging.debug( 'Starting to project input vector FC to the spatial reference of the output raster' ) out_projected_fc = os.path.join(temp_dir, 'temp.gdb', 'src_prj_to_ras_sr') arcpy.Project_management(input_fc, out_projected_fc, sr) util.add_field_and_calculate(out_projected_fc, 'ras_val', 'SHORT', '', 1, self.gfw_env) # Get cell size of output cell_size = int( arcpy.GetRasterProperties_management( self.vector_to_raster_output, 'CELLSIZEX').getOutput(0)) arcpy.env.pyramid = "NONE" arcpy.env.snapRaster = self.vector_to_raster_output logging.debug('Rasterizing and outputting as tif') out_raster = os.path.join(temp_dir, 'out.tif') arcpy.PolygonToRaster_conversion(out_projected_fc, 'ras_val', out_raster, "CELL_CENTER", "", cell_size) # Stop service that has a lock on the raster service = r'image_services/analysis' # arcgis_server.set_service_status(service, 'stop') logging.debug( 'Sleeping for 10 seconds to let the lock files resolve themselves' ) time.sleep(10) logging.debug('Copying raster {0} to output {1}'.format( out_raster, self.vector_to_raster_output)) arcpy.Delete_management(self.vector_to_raster_output) # Move all related tif files to final destination # Much faster than using CopyRaster_management-- just need to physically move the files src_dir = os.path.dirname(out_raster) src_file_name = os.path.splitext(os.path.basename(out_raster))[0] out_dir = os.path.dirname(self.vector_to_raster_output) out_file_name = os.path.splitext( os.path.basename(self.vector_to_raster_output))[0] for extension in [ '.tif', '.tfw', '.tif.aux.xml', '.tif.vat.cpg', '.tif.vat.dbf', '.tif.xml' ]: src_file = os.path.join(src_dir, src_file_name + extension) out_file = os.path.join(out_dir, out_file_name + extension) try: shutil.move(src_file, out_file) except IOError: print 'No such file {0}'.format(src_file) # Restart the service after we're finished # arcgis_server.set_service_status(service, 'start') else: pass