def _downsample_dir(dir_src: str) -> None: filenames_srcs = [fn for fn in os.listdir(dir_src) if fn.endswith('.tif')] for filename_src in tqdm(filenames_srcs, desc='Downsampling rasters'): filepath_src = os.path.join(dir_src, filename_src) filepath_lock = os.path.join(dir_src, filename_src + '.lock') try: file_lock = open(filepath_lock, 'x') except OSError: continue try: for pct in DOWNSAMPLE_PCTS: dir_dest = os.path.join(os.path.dirname(dir_src), paths.SUBDIR_DATA_TRAIN_DOWNSAMPLE.format(pct)) filename_dest = re.sub('.tif', '_{}.tif'.format(pct), filename_src) filepath_dest = os.path.join(dir_dest, filename_dest) if os.path.exists(filepath_dest): continue if not os.path.exists(dir_dest): try: os.makedirs(dir_dest) except: pass command = 'gdal_translate -outsize {pct}% {pct}% -r nearest {filepath_src} {filepath_dest}'.format( pct=pct, filepath_src=filepath_src, filepath_dest=filepath_dest) gdal_command_line.run_gdal_command(command, _logger) except Exception as error_: raise error_ finally: file_lock.close() os.remove(filepath_lock)
def _submit_uploads(manifests: List[dict]) -> None: tmp_filename = 'tmp_manifest.json' for manifest in manifests: with open(tmp_filename, 'w') as file_: json.dump(manifest, file_) command = 'earthengine upload image --manifest {}'.format(tmp_filename) gdal_command_line.run_gdal_command(command) os.remove(tmp_filename)
def _reproject_shapefiles() -> None: filenames_raw_polys = [ filename for filename in os.listdir(paths.DIR_DATA_TRAIN_MP_RAW) if filename.endswith('.shp') and not filename.endswith( '_responses.shp') and not filename.endswith('_3857.shp') ] for filename_raw in tqdm(filenames_raw_polys, desc='Reproject shapefiles'): filepath_raw = os.path.join(paths.DIR_DATA_TRAIN_MP_RAW, filename_raw) filename_reproj = re.sub('.shp', '_3857.shp', filename_raw) filepath_reproj = os.path.join(paths.DIR_DATA_TRAIN_MP_RAW, filename_reproj) if os.path.exists(filepath_reproj): continue command = 'ogr2ogr -t_srs EPSG:3857 {reproj} {raw}'.format( reproj=filepath_reproj, raw=filepath_raw) gdal_command_line.run_gdal_command(command, _logger)
def remove_feature_rasters_alpha_band() -> None: filenames_raw = [ fn for fn in os.listdir(paths.DIR_DATA_TRAIN_FEATURES_RAW) if fn.endswith('features.tif') ] for filename_raw in tqdm(filenames_raw, desc='Remove feature rasters alpha band'): filepath_lock = os.path.join(paths.DIR_DATA_TRAIN_FEATURES_RAW, filename_raw + '.lock') filepath_raw = os.path.join(paths.DIR_DATA_TRAIN_FEATURES_RAW, filename_raw) filename_tmp = re.sub('features.tif', 'features_tmp.tif', filename_raw) filepath_tmp = os.path.join(paths.DIR_DATA_TRAIN_FEATURES_RAW, filename_tmp) filepath_clean = os.path.join(paths.DIR_DATA_TRAIN_FEATURES_CLEAN, filename_raw) # Skip if output file already exists or if lock file exists if os.path.exists(filepath_clean) or os.path.exists(filepath_lock): continue try: file_lock = open(filepath_lock, 'x') except OSError: continue try: # Write in nodata values command = 'gdal_calc.py -A {filepath_raw} --allBands=A -B {filepath_raw} --B_band=4 ' + \ '--outfile={filepath_tmp} --NoDataValue=-9999 --type=Int16 --co=COMPRESS=DEFLATE ' + \ '--co=TILED=YES --overwrite --calc="A * (B == 255) + -9999 * (B == 0)"' command = command.format(filepath_raw=filepath_raw, filepath_tmp=filepath_tmp) gdal_command_line.run_gdal_command(command, _logger) # Remove alpha band options_removed = gdal.TranslateOptions( bandList=[1, 2, 3], outputType=gdal.GDT_Int16, creationOptions=['COMPRESS=DEFLATE', 'TILED=YES']) gdal.Translate(filepath_clean, filepath_tmp, options=options_removed) except Exception as error_: raise error_ finally: if os.path.exists(filepath_tmp): os.remove(filepath_tmp) file_lock.close() os.remove(filepath_lock)
def downsample_global_rasters() -> None: _logger.info('Downsample rasters') filenames = sorted([ fn for fn in os.listdir(paths.DIR_DATA_GLOBAL) if re.search('L15-\d{4}E-\d{4}N.tif', fn) ]) for filename in tqdm(filenames, desc='Downsampling rasters'): _logger.debug('Downsample raster: {}'.format(filename)) filepath_in = os.path.join(paths.DIR_DATA_GLOBAL, filename) filepath_lock = filepath_in + '.lock' if os.path.exists(filepath_lock) or not os.path.exists( filepath_in) or os.path.exists(filepath_out): continue try: file_lock = open(filepath_lock, 'x') except OSError: continue try: for pct in _DOWNSAMPLE_PCTS: filename_out = re.sub('.tif', '_{}.tif'.format(pct), filename) dir_out = os.path.join( paths.DIR_DATA_GLOBAL, paths.SUBDIR_DATA_TRAIN_DOWNSAMPLE.format(pct)) filepath_out = os.path.join(dir_out, filename_out) if not os.path.exists(dir_out): try: os.makedirs(dir_out) except OSError: pass command = 'gdal_translate -outsize {pct}% {pct}% -r bilinear {path_in} {path_out}'.format( pct=pct, path_in=filepath_in, path_out=filepath_out) gdal_command_line.run_gdal_command(command, _logger) os.remove(filepath_in) except Exception as error_: raise error_ finally: file_lock.close() os.remove(filepath_lock)
def create_sampling_boundary_shapefiles() -> None: raise AssertionError('This script has not been tested since being updated, be careful') _logger.info('Creating sampling boundary shapefiles for original UQ training data') _assert_encoding_assumptions_hold() # Get list of completed responses rasters tmp_filepath_reef_raster = os.path.join(paths.DIR_DATA_TRAIN_RAW, 'tmp_reef_only.tif') tmp_filepath_reef_outline = os.path.join(paths.DIR_DATA_TRAIN_RAW, 'tmp_reef_outline.shp') basename_reef_outline = os.path.splitext(os.path.basename(tmp_filepath_reef_outline))[0] filepaths_responses = sorted([ os.path.join(paths.DIR_DATA_TRAIN_CLEAN, filename) for filename in os.listdir(paths.DIR_DATA_TRAIN_CLEAN) if filename.endswith('_responses.tif') ]) for idx_responses, filepath_responses in enumerate(filepaths_responses): _logger.debug('Creating boundaries for response file {} of {}'.format(idx_responses, len(filepaths_responses))) filepath_boundary = re.sub('responses.tif', 'boundaries.shp', filepath_responses) if os.path.exists(filepath_boundary): _logger.debug('Boundary file already exists at: {}'.format(filepath_boundary)) continue _logger.debug('Creating boundary file at: {}'.format(filepath_boundary)) # Get raster of only reef areas _logger.debug('Creating reef-only raster') min_reef_value = min(encodings.MAPPINGS[encodings.REEF_TOP], encodings.MAPPINGS[encodings.NOT_REEF_TOP]) command = 'gdal_calc.py -A {filepath_responses} --outfile={filepath_reef} --NoDataValue=-9999 ' + \ '--calc="1*(A>={min_value}) + -9999*(A<{min_value})"' command = command.format( filepath_responses=filepath_responses, filepath_reef=tmp_filepath_reef_raster, min_reef_value=min_reef_value) gdal_command_line.run_gdal_command(command, _logger) # Get shapefile of reef outline _logger.debug('Creating reef outline shapefile') command = 'gdal_polygonize.py {} {}'.format(tmp_filepath_reef_raster, tmp_filepath_reef_outline) gdal_command_line.run_gdal_command(command, _logger) # Get shapefile of sampling boundaries by buffering reef outline _logger.debug('Creating buffered outline for boundary file') command = 'ogr2ogr -f "ESRI Shapefile" {filepath_boundary} {filepath_outline} -dialect sqlite ' + \ '-sql "select ST_buffer(geometry, 64) as geometry from {basename_outline}"' command = command.format( filepath_boundary=filepath_boundary, filepath_outline=tmp_filepath_reef_outline, basename_outline=basename_reef_outline ) gdal_command_line.run_gdal_command(command, _logger) # Clean up _logger.debug('Remove temporary files') os.remove(tmp_filepath_reef_raster) for filename_outline in os.listdir(paths.DIR_DATA_TRAIN_RAW): if not re.search(basename_reef_outline, filename_outline): continue os.remove(os.path.join(paths.DIR_DATA_TRAIN_RAW, filename_outline))
def create_sampling_boundary_shapefiles() -> None: _logger.info('Creating sampling boundary shapefiles for supplemental training data') # Get list of supplemental shapefiles tmp_filepath_raster = os.path.join(paths.DIR_DATA_TRAIN_RAW, 'tmp_raster_valid.tif') tmp_filepath_polygon = os.path.join(paths.DIR_DATA_TRAIN_RAW, 'tmp_outline_valid.shp') basename_polygon = os.path.splitext(os.path.basename(tmp_filepath_polygon))[0] filepaths_shapes = sorted([ os.path.join(paths.DIR_DATA_TRAIN_CLEAN, filename) for filename in os.listdir(paths.DIR_DATA_TRAIN_CLEAN) if filename.endswith('_model_class.shp') ]) for filepath_shape in tqdm(filepaths_shapes, desc='Create sampling boundary shapefiles'): filepath_boundary = re.sub('.shp', 'boundaries.shp', filepath_shape) if os.path.exists(filepath_boundary): _logger.debug('Boundary file already exists at: {}'.format(filepath_boundary)) continue _logger.debug('Creating boundary file at: {}'.format(filepath_boundary)) _logger.debug('Creating valid sampling area raster') command = 'gdal_rasterize -burn 1 -tr 5 5 -a_nodata -9999 -init -9999 ' + \ '{filepath_shape} {tmp_filepath_raster}' command = command.format(filepath_shape=filepath_shape, tmp_filepath_raster=tmp_filepath_raster) gdal_command_line.run_gdal_command(command, _logger) _logger.debug('Creating valid sampling area outline') command = 'gdal_polygonize.py {} {}'.format(tmp_filepath_raster, tmp_filepath_polygon) gdal_command_line.run_gdal_command(command, _logger) _logger.debug('Creating buffered outline for boundary file') command = 'ogr2ogr -f "ESRI Shapefile" {filepath_boundary} {tmp_filepath_polygon} -dialect sqlite ' + \ '-sql "select ST_buffer(geometry, 64) as geometry from {basename_polygon}"' command = command.format( filepath_boundary=filepath_boundary, tmp_filepath_polygon=tmp_filepath_polygon, basename_polygon=basename_polygon ) gdal_command_line.run_gdal_command(command, _logger) _logger.debug('Remove temporary files') os.remove(tmp_filepath_raster) for extension in ('.dbf', '.prj', '.shp', '.shx'): os.remove(os.path.join(paths.DIR_DATA_TRAIN_RAW, basename_polygon + extension))
def create_sampling_boundary_shapefiles() -> None: _logger.info( 'Creating sampling boundary shapefiles for Millennium Project training data' ) filenames_responses = sorted([ filename for filename in os.listdir(paths.DIR_DATA_TRAIN_MP_CLEAN) if filename.endswith('_responses_custom.tif') ]) desc = 'Create sampling boundary shapefiles' for idx_filepath, filename_response in enumerate( tqdm(filenames_responses, desc=desc)): filepath_response = os.path.join(paths.DIR_DATA_TRAIN_MP_CLEAN, filename_response) filename_sqlvalid = re.sub('-', '_', os.path.basename(filepath_response)) tmp_filename_raster = re.sub('_responses_custom.tif', '_tmp_noland.tif', filename_sqlvalid) tmp_filepath_raster = os.path.join(paths.DIR_DATA_TRAIN_MP_BOUNDS, tmp_filename_raster) tmp_filename_outline = re.sub('_responses_custom.tif', '_tmp_noland.shp', filename_sqlvalid) tmp_filepath_outline = os.path.join(paths.DIR_DATA_TRAIN_MP_BOUNDS, tmp_filename_outline) basename_outline = os.path.splitext( os.path.basename(tmp_filepath_outline))[0] filename_boundary = re.sub('_responses_custom.tif', '_boundaries.shp', filename_response) filepath_boundary = os.path.join(paths.DIR_DATA_TRAIN_MP_BOUNDS, filename_boundary) filepath_lock = filepath_boundary + '.lock' if os.path.exists(filepath_boundary): _logger.debug('Boundary file already exists at: {}'.format( filepath_boundary)) continue if os.path.exists(filepath_lock): _logger.debug('Boundary file already being processed: {}'.format( filepath_boundary)) continue try: file_lock = open(filepath_lock, 'x') except OSError: continue _logger.debug('Creating boundary file {} ({} total): {}'.format( idx_filepath, len(filenames_responses), filepath_response)) try: # Get raster of areas we care about, those that are not land _logger.debug('Creating non-land raster') code_land = encodings_mp.CODE_LAND command = 'gdal_calc.py -A {filepath_response} --outfile={tmp_raster} --NoDataValue=-9999 ' + \ '--calc="1*(A!={code_land}) + -9999*(A=={code_land})"' command = command.format(filepath_response=filepath_response, tmp_raster=tmp_filepath_raster, code_land=code_land) gdal_command_line.run_gdal_command(command, _logger) # Get shapefile for outline of areas we care about _logger.debug('Creating non-land outline shapefile') command = 'gdal_polygonize.py {} {}'.format( tmp_filepath_raster, tmp_filepath_outline) gdal_command_line.run_gdal_command(command, _logger) # Get shapefile of sampling boundaries by buffering reef outline _logger.debug('Creating buffered outline for boundary file') command = 'ogr2ogr -f "ESRI Shapefile" {filepath_boundary} {tmp_outline} -dialect sqlite ' + \ '-sql "select ST_buffer(geometry, 64) as geometry from {basename_outline}"' command = command.format(filepath_boundary=filepath_boundary, tmp_outline=tmp_filepath_outline, basename_outline=basename_outline) gdal_command_line.run_gdal_command(command, _logger) except Exception as error_: raise error_ finally: if os.path.exists(tmp_filepath_raster): os.remove(tmp_filepath_raster) for filename in os.listdir(os.path.dirname(tmp_filepath_outline)): if not re.search(basename_outline, filename): continue os.remove( os.path.join(os.path.dirname(tmp_filepath_outline), filename)) file_lock.close() os.remove(filepath_lock)
def create_millennium_project_quad_rasters_custom_classes() -> None: _logger.info('Create Millennium Project response quad rasters') if not os.path.exists(paths.DIR_DATA_TRAIN_MP_CLEAN): os.makedirs(paths.DIR_DATA_TRAIN_MP_CLEAN) filenames_polys = [ filename for filename in os.listdir(paths.DIR_DATA_TRAIN_MP_RAW) if filename.startswith('L15-') and filename.endswith('responses.shp') ] missing_features = list() for idx_filename, filename_poly in enumerate( tqdm(filenames_polys, desc='Create Millennium Project rasters')): _logger.debug('Create raster {} ({} total): {}'.format( idx_filename, len(filenames_polys), filename_poly)) # Set filepaths filepath_src = os.path.join(paths.DIR_DATA_TRAIN_MP_RAW, filename_poly) filename_custom = re.sub('responses.shp', 'responses_tmp.shp', filename_poly) filepath_custom = os.path.join(paths.DIR_DATA_TRAIN_MP_RAW, filename_custom) filename_dest = re.sub('.shp', '_custom.tif', filename_poly) filepath_dest = os.path.join(paths.DIR_DATA_TRAIN_MP_CLEAN, filename_dest) filepath_lock = filepath_dest + '.lock' if os.path.exists(filepath_dest) or os.path.exists(filepath_lock): continue try: file_lock = open(filepath_lock, 'x') except OSError: continue try: # Try to find existing features file, may be either raw or clean, but also may not be available from Vulcan filename_features = re.search( r'L15-\d{4}E-\d{4}N', filename_poly).group() + '_features.tif' filepath_features = os.path.join(paths.DIR_DATA_TRAIN_FEATURES, filename_features) if not os.path.exists(filepath_features): filepath_features = os.path.join( paths.DIR_DATA_TRAIN_FEATURES_CLEAN, filename_features) if not os.path.exists(filepath_features): _logger.warning( 'Features file not available in raw or clean dirs: {}'. format(filename_features)) missing_features.append(filename_features) continue # Create shapefile with custom classes crs = fiona.crs.from_epsg(SHP_EPSG) with fiona.open(filepath_custom, 'w', driver=SHP_DRIVER, crs=crs, schema=SHP_SCHEMA) as file_custom: for original_feature in fiona.open(filepath_src): original_code = original_feature['properties']['L4_CODE'] custom_code = encodings_mp.MAPPINGS_CUSTOM[original_code] custom_feature = { 'properties': { 'custom': custom_code }, 'geometry': original_feature['geometry'] } file_custom.write(custom_feature) # Get rasterize parameters raster_features = gdal.Open(filepath_features) cols = raster_features.RasterXSize rows = raster_features.RasterYSize llx, xres, _, y0, _, yres = raster_features.GetGeoTransform() urx = llx + cols * xres y1 = y0 + rows * yres lly = min([y0, y1]) ury = max([y0, y1]) # Rasterize command = 'gdal_rasterize -ot Int16 -co COMPRESS=DEFLATE -co TILED=YES -a_nodata -9999 -init -9999 ' \ '-a custom -te {llx} {lly} {urx} {ury} -tr {xres} {yres} {filepath_custom} {filepath_dest}' command = command.format(llx=llx, lly=lly, urx=urx, ury=ury, xres=xres, yres=yres, filepath_custom=filepath_custom, filepath_dest=filepath_dest) gdal_command_line.run_gdal_command(command, _logger) except Exception as error_: raise error_ finally: for filename in os.listdir(os.path.dirname(filepath_custom)): if re.search(os.path.splitext(filename_custom)[0], filename): os.remove( os.path.join(os.path.dirname(filepath_custom), filename)) file_lock.close() os.remove(filepath_lock) assert not missing_features, 'Missing features files: {}'.format( ', '.join(missing_features))
def rasterize_response_quads() -> None: raise AssertionError( 'This script has not been tested since being updated, be careful') _assert_encoding_assumptions_hold() filenames = [ filename for filename in os.listdir(paths.DIR_DATA_TRAIN_RAW) if filename.endswith('.shp') ] for idx_filename, filename in enumerate(filenames): print('\rRasterizing shapefile {} of {}'.format( 1 + idx_filename, len(filenames))) # Get quad and filepaths quad = re.search(r'L15-\d{4}E-\d{4}N', filename).group() filepath_features = os.path.join( paths.DIR_DATA_TRAIN_CLEAN, quad + data_bucket.FILENAME_SUFFIX_FEATURES) filepath_source_responses = os.path.join(paths.DIR_DATA_TRAIN_RAW, filename) filepath_dest_lwr = re.sub('features.tif', 'responses_lwr.tif', filepath_features) filepath_dest_lwrn = re.sub('features.tif', 'responses_lwrn.tif', filepath_features) assert os.path.exists( filepath_features ), 'Download all feature quads before rasterizing response quads' # Get rasterize parameters raster_features = gdal.Open(filepath_features) srs = osr.SpatialReference(wkt=raster_features.GetProjection()) cols = raster_features.RasterXSize rows = raster_features.RasterYSize llx, xres, _, y0, _, yres = raster_features.GetGeoTransform() urx = llx + cols * xres y1 = y0 + rows * yres lly = min([y0, y1]) ury = max([y0, y1]) # Rasterize into Land-Water-Reef-NotReef options_rasterize = gdal.RasterizeOptions( outputType=gdal.GDT_Int16, creationOptions=['COMPRESS=DEFLATE', 'TILED=YES'], outputBounds=[llx, lly, urx, ury], outputSRS=srs, xRes=xres, yRes=yres, noData=-9999, initValues=-9999, attribute='class_code') raster_out = gdal.Rasterize(filepath_dest_lwrn, filepath_source_responses, options=options_rasterize) del raster_out # Remove cloud-shade and unknown classes. Unknown classes could be anything from water to reef to clouds, while # cloud-shade is not reliable as the map was created with the analytical mosaic rather than the visual mosaic. min_nodata = min(encodings.MAPPINGS[encodings.CLOUD_SHADE], encodings.MAPPINGS[encodings.UNKNOWN]) command = 'gdal_calc.py -A {filepath} --outfile {filepath} --NoDataValue=-9999 --overwrite ' + \ '--calc="A * (A < {min_nodata}) + -9999 * (A >= {min_nodata})"' command = command.format(filepath=filepath_dest_lwrn, min_nodata=min_nodata) gdal_command_line.run_gdal_command(command, _logger) # Create Land-Water-Reef val_reef = encodings.MAPPINGS[encodings.REEF_TOP] val_notreef = encodings.MAPPINGS[encodings.NOT_REEF_TOP] command = 'gdal_calc.py -A {filepath_lwrn} --outfile={filepath_lwr} --NoDataValue=-9999 --overwrite ' + \ '--calc="A * (A != {val_notreef}) + {val_reef} * (A == {val_notreef})"' command = command.format(filepath_lwrn=filepath_dest_lwrn, filepath_lwr=filepath_dest_lwr, val_notreef=val_notreef, val_reef=val_reef) gdal_command_line.run_gdal_command(command, _logger)
] for fn in fns: quad = re.search('L15-\d{4}E-\d{4}N', fn).group() dest = os.path.join('../../responses_mp_supp', quad + '_water_cloud.tif') if not os.path.exists(fn): continue shutil.copy(fn, dest) # Convert land to 2000 fns = [fn for fn in os.listdir() if fn.endswith('_land.tif')] for fn in fns: dest = os.path.join('../clean', os.path.basename(fn)) command = 'gdal_calc.py -A {fn} --outfile={dest} --calc="2000*(A==1)+-9999*(A!=1)" --NoDataValue=-9999 --co=TILED=YES --co=COMPRESS=DEFLATE'.format( fn=fn, dest=dest) gdal_command_line.run_gdal_command(command) # Convert water to 2001 fns = [fn for fn in os.listdir() if fn.endswith('_water.tif')] for fn in fns: dest = os.path.join('../clean', os.path.basename(fn)) command = 'gdal_calc.py -A {fn} --outfile={dest} --calc="2001*(A==2)+-9999*(A!=2)" --NoDataValue=-9999 --co=TILED=YES --co=COMPRESS=DEFLATE'.format( fn=fn, dest=dest) gdal_command_line.run_gdal_command(command) # Convert clouds to 2002 fns = [fn for fn in os.listdir() if fn.endswith('_water_cloud.tif')] for fn in fns: dest = os.path.join('../clean', os.path.basename(fn)) command = 'gdal_calc.py -A {fn} --outfile={dest} --calc="2001*(A==2)+2002*(A==23)+-9999*numpy.logical_and(A!=2, A!=23)" --NoDataValue=-9999 --co=TILED=YES --co=COMPRESS=DEFLATE'.format( fn=fn, dest=dest)