def update_cache(map_service_path, scratch_workspace, gfw_env):
logging.debug('Updating tiles for cached map service {0}'.format(map_service_path))
# Start the service on localhost-- should be shut off when not in use
manage_service('dev', map_service_path, 'start')
source_mxd, local_cache_dir = find_src_mxd_and_cache_dir(map_service_path)
logging.debug("Found source MXD: {0}".format(source_mxd))
output_dir = util.create_temp_dir(scratch_workspace)
# Zoom levels 0 - 6
scale_aoi = "591657527.591555;295828763.795777;147914381.897889;73957190.948944;" \
"36978595.474472;18489297.737236;9244648.868618"
min_scale = scale_aoi.split(';')[0]
max_scale = scale_aoi.split(';')[-1]
logging.debug("Generating tiles . . . ")
cache_dir_name = 'cache'
arcpy.ManageTileCache_management(output_dir, "RECREATE_ALL_TILES", cache_dir_name, source_mxd,
"ARCGISONLINE_SCHEME", "", scale_aoi, "", "", min_scale, max_scale)
logging.debug("Copying to local and production cache directories")
src_cache_dir = os.path.join(output_dir, cache_dir_name, 'Layers')
if gfw_env == 'prod':
push_to_production(src_cache_dir, local_cache_dir, map_service_path)
else:
logging.debug("Nothing pushed to prod dir; just testing cache generation process")
logging.debug('Cached tiles are here: {0}'.format(local_cache_dir))
# Stop the map service-- no need for it to be serving on the DM machine
manage_service('dev', map_service_path, 'stop')
def vector_to_raster(self, input_fc):
if self.vector_to_raster_output:
temp_dir = util.create_temp_dir(self.scratch_workspace)
arcpy.CreateFileGDB_management(temp_dir, 'temp.gdb')
# Get spatial reference of output
sr = arcpy.Describe(self.vector_to_raster_output).spatialReference
logging.debug('Starting to project input vector FC to the spatial reference of the output raster')
out_projected_fc = os.path.join(temp_dir, 'temp.gdb', 'src_prj_to_ras_sr')
arcpy.Project_management(input_fc, out_projected_fc, sr)
util.add_field_and_calculate(out_projected_fc, 'ras_val', 'SHORT', '', 1, self.gfw_env)
# Get cell size of output
cell_size = int(arcpy.GetRasterProperties_management(self.vector_to_raster_output,
'CELLSIZEX').getOutput(0))
arcpy.env.pyramid = "NONE"
arcpy.env.snapRaster = self.vector_to_raster_output
logging.debug('Rasterizing and outputting as tif')
out_raster = os.path.join(temp_dir, 'out.tif')
arcpy.PolygonToRaster_conversion(out_projected_fc, 'ras_val', out_raster, "CELL_CENTER", "", cell_size)
# Stop service that has a lock on the raster
service = r'image_services/analysis'
# arcgis_server.set_service_status(service, 'stop')
logging.debug('Sleeping for 10 seconds to let the lock files resolve themselves')
time.sleep(10)
logging.debug('Copying raster {0} to output {1}'.format(out_raster, self.vector_to_raster_output))
arcpy.Delete_management(self.vector_to_raster_output)
# Move all related tif files to final destination
# Much faster than using CopyRaster_management-- just need to physically move the files
src_dir = os.path.dirname(out_raster)
src_file_name = os.path.splitext(os.path.basename(out_raster))[0]
out_dir = os.path.dirname(self.vector_to_raster_output)
out_file_name = os.path.splitext(os.path.basename(self.vector_to_raster_output))[0]
for extension in ['.tif', '.tfw', '.tif.aux.xml', '.tif.vat.cpg', '.tif.vat.dbf', '.tif.xml']:
src_file = os.path.join(src_dir, src_file_name + extension)
out_file = os.path.join(out_dir, out_file_name + extension)
try:
shutil.move(src_file, out_file)
except IOError:
print 'No such file {0}'.format(src_file)
# Restart the service after we're finished
# arcgis_server.set_service_status(service, 'start')
else:
pass
def update_cache(map_service_path, scratch_workspace):
logging.debug('Updating tiles for cached map service {0}'.format(map_service_path))
# Start the service on localhost-- should be shut off when not in use
manage_service('dev', map_service_path, 'start')
source_mxd, local_cache_dir = find_src_mxd_and_cache_dir(map_service_path)
logging.debug("Found source MXD: {0}".format(source_mxd))
output_dir = util.create_temp_dir(scratch_workspace)
# Zoom levels 0 - 6
scale_aoi = "591657527.591555;295828763.795777;147914381.897889;73957190.948944;" \
"36978595.474472;18489297.737236;9244648.868618"
min_scale = scale_aoi.split(';')[0]
max_scale = scale_aoi.split(';')[-1]
logging.debug("Generating tiles . . . ")
cache_dir_name = 'cache'
arcpy.ManageTileCache_management(output_dir, "RECREATE_ALL_TILES", cache_dir_name, source_mxd,
"ARCGISONLINE_SCHEME", "", scale_aoi, "", "", min_scale, max_scale)
logging.debug("Copying to local and production cache directories")
src_cache_dir = os.path.join(output_dir, cache_dir_name, 'Layers')
push_to_production(src_cache_dir, local_cache_dir, map_service_path)
# Stop the map service-- no need for it to be serving on the DM machine
manage_service('dev', map_service_path, 'stop')
def main():
parser = argparse.ArgumentParser(description='QC loss and extent stats')
parser.add_argument('--number-of-tiles',
'-n',
help='number of tiles to run QC',
default=10,
type=int,
required=True)
parser.add_argument('--grid-resolution',
'-g',
help='grid resolution of source',
type=int,
default=10,
choices=(10, 0.25))
parser.add_argument('--s3-poly-dir',
'-s',
help='input poly directory for intersected TSVs',
required=True)
parser.add_argument('--thread-count',
'-c',
type=int,
default=mp.cpu_count())
parser.add_argument('--test', dest='test', action='store_true')
args = parser.parse_args()
util.start_logging()
# load gadm28 - required to determine which gadm28 boundaries are completely within a 10x10 tile
load_gadm28.load('s3://gfw2-data/alerts-tsv/gis_source/adm2_final.zip')
if args.test:
args.number_of_tiles = 1
tile_list = s3_list_tiles.pull_random(args.s3_poly_dir,
args.number_of_tiles)
print tile_list
temp_dir = util.create_temp_dir()
process_list = []
for tile in tile_list:
process_list.append((tile, args.s3_poly_dir, temp_dir))
util.exec_multiprocess(qc_output_tile, process_list, args.test,
args.thread_count)
qc.check_results()
def update_cache(map_service_path, scratch_workspace, gfw_env):
logging.debug("Updating tiles for cached map service {0}".format(map_service_path))
# Start the service on localhost-- should be shut off when not in use
manage_service("dev", map_service_path, "start")
source_mxd, local_cache_dir = find_src_mxd_and_cache_dir(map_service_path)
logging.debug("Found source MXD: {0}".format(source_mxd))
output_dir = util.create_temp_dir(scratch_workspace)
# Zoom levels 0 - 6
scale_aoi = (
"591657527.591555;295828763.795777;147914381.897889;73957190.948944;"
"36978595.474472;18489297.737236;9244648.868618"
)
min_scale = scale_aoi.split(";")[0]
max_scale = scale_aoi.split(";")[-1]
logging.debug("Generating tiles . . . ")
cache_dir_name = "cache"
arcpy.ManageTileCache_management(
output_dir,
"RECREATE_ALL_TILES",
cache_dir_name,
source_mxd,
"ARCGISONLINE_SCHEME",
"",
scale_aoi,
"",
"",
min_scale,
max_scale,
)
logging.debug("Copying to local and production cache directories")
src_cache_dir = os.path.join(output_dir, cache_dir_name, "Layers")
if gfw_env == "PROD":
push_to_production(src_cache_dir, local_cache_dir, map_service_path)
else:
logging.debug("Nothing pushed to PROD dir; just testing cache generation process")
logging.debug("Cached tiles are here: {0}".format(local_cache_dir))
# Stop the map service-- no need for it to be serving on the DM machine
manage_service("dev", map_service_path, "stop")
def vector_to_raster(self, input_fc):
if self.vector_to_raster_output:
temp_dir = util.create_temp_dir(self.scratch_workspace)
arcpy.CreateFileGDB_management(temp_dir, 'temp.gdb')
# Get spatial reference of output
sr = arcpy.Describe(self.vector_to_raster_output).spatialReference
logging.debug('Starting to project input vector FC to the spatial reference of the output raster')
out_projected_fc = os.path.join(temp_dir, 'temp.gdb', 'src_prj_to_ras_sr')
arcpy.Project_management(input_fc, out_projected_fc, sr)
util.add_field_and_calculate(out_projected_fc, 'ras_val', 'SHORT', '', 1, self.gfw_env)
# Get cell size of output
cell_size = int(arcpy.GetRasterProperties_management(self.vector_to_raster_output,
'CELLSIZEX').getOutput(0))
arcpy.env.pyramid = "NONE"
arcpy.env.snapRaster = self.vector_to_raster_output
logging.debug('Rasterizing and outputting as tif')
out_raster = os.path.join(temp_dir, 'out.tif')
arcpy.PolygonToRaster_conversion(out_projected_fc, 'ras_val', out_raster, "CELL_CENTER", "", cell_size)
logging.debug('Copying raster {0} to output {1}'.format(out_raster, self.vector_to_raster_output))
arcpy.Delete_management(self.vector_to_raster_output)
# Move all related tif files to final destination
# Much faster than using CopyRaster_management-- just need to physically move the files
src_dir = os.path.dirname(out_raster)
src_file_name = os.path.splitext(os.path.basename(out_raster))[0]
out_dir = os.path.dirname(self.vector_to_raster_output)
out_file_name = os.path.splitext(os.path.basename(self.vector_to_raster_output))[0]
for extension in ['.tif', '.tfw', '.tif.aux.xml', '.tif.vat.cpg', '.tif.vat.dbf', '.tif.xml']:
src_file = os.path.join(src_dir, src_file_name + extension)
out_file = os.path.join(out_dir, out_file_name + extension)
shutil.move(src_file, out_file)
else:
pass
def vector_to_raster(self, input_fc):
if self.vector_to_raster_output:
temp_dir = util.create_temp_dir(self.scratch_workspace)
arcpy.CreateFileGDB_management(temp_dir, 'temp.gdb')
# Get spatial reference of output
sr = arcpy.Describe(self.vector_to_raster_output).spatialReference
logging.debug(
'Starting to project input vector FC to the spatial reference of the output raster'
)
out_projected_fc = os.path.join(temp_dir, 'temp.gdb',
'src_prj_to_ras_sr')
arcpy.Project_management(input_fc, out_projected_fc, sr)
util.add_field_and_calculate(out_projected_fc, 'ras_val', 'SHORT',
'', 1, self.gfw_env)
# Get cell size of output
cell_size = int(
arcpy.GetRasterProperties_management(
self.vector_to_raster_output, 'CELLSIZEX').getOutput(0))
arcpy.env.pyramid = "NONE"
arcpy.env.snapRaster = self.vector_to_raster_output
logging.debug('Rasterizing and outputting as tif')
out_raster = os.path.join(temp_dir, 'out.tif')
arcpy.PolygonToRaster_conversion(out_projected_fc, 'ras_val',
out_raster, "CELL_CENTER", "",
cell_size)
# Stop service that has a lock on the raster
service = r'image_services/analysis'
# arcgis_server.set_service_status(service, 'stop')
logging.debug(
'Sleeping for 10 seconds to let the lock files resolve themselves'
)
time.sleep(10)
logging.debug('Copying raster {0} to output {1}'.format(
out_raster, self.vector_to_raster_output))
arcpy.Delete_management(self.vector_to_raster_output)
# Move all related tif files to final destination
# Much faster than using CopyRaster_management-- just need to physically move the files
src_dir = os.path.dirname(out_raster)
src_file_name = os.path.splitext(os.path.basename(out_raster))[0]
out_dir = os.path.dirname(self.vector_to_raster_output)
out_file_name = os.path.splitext(
os.path.basename(self.vector_to_raster_output))[0]
for extension in [
'.tif', '.tfw', '.tif.aux.xml', '.tif.vat.cpg',
'.tif.vat.dbf', '.tif.xml'
]:
src_file = os.path.join(src_dir, src_file_name + extension)
out_file = os.path.join(out_dir, out_file_name + extension)
try:
shutil.move(src_file, out_file)
except IOError:
print 'No such file {0}'.format(src_file)
# Restart the service after we're finished
# arcgis_server.set_service_status(service, 'start')
else:
pass
