def _downsample_dir(dir_src: str) -> None:
filenames_srcs = [fn for fn in os.listdir(dir_src) if fn.endswith('.tif')]
for filename_src in tqdm(filenames_srcs, desc='Downsampling rasters'):
filepath_src = os.path.join(dir_src, filename_src)
filepath_lock = os.path.join(dir_src, filename_src + '.lock')
try:
file_lock = open(filepath_lock, 'x')
except OSError:
continue
try:
for pct in DOWNSAMPLE_PCTS:
dir_dest = os.path.join(os.path.dirname(dir_src), paths.SUBDIR_DATA_TRAIN_DOWNSAMPLE.format(pct))
filename_dest = re.sub('.tif', '_{}.tif'.format(pct), filename_src)
filepath_dest = os.path.join(dir_dest, filename_dest)
if os.path.exists(filepath_dest):
continue
if not os.path.exists(dir_dest):
try:
os.makedirs(dir_dest)
except:
pass
command = 'gdal_translate -outsize {pct}% {pct}% -r nearest {filepath_src} {filepath_dest}'.format(
pct=pct, filepath_src=filepath_src, filepath_dest=filepath_dest)
gdal_command_line.run_gdal_command(command, _logger)
except Exception as error_:
raise error_
finally:
file_lock.close()
os.remove(filepath_lock)
def _submit_uploads(manifests: List[dict]) -> None:
tmp_filename = 'tmp_manifest.json'
for manifest in manifests:
with open(tmp_filename, 'w') as file_:
json.dump(manifest, file_)
command = 'earthengine upload image --manifest {}'.format(tmp_filename)
gdal_command_line.run_gdal_command(command)
os.remove(tmp_filename)
def _reproject_shapefiles() -> None:
filenames_raw_polys = [
filename for filename in os.listdir(paths.DIR_DATA_TRAIN_MP_RAW)
if filename.endswith('.shp') and not filename.endswith(
'_responses.shp') and not filename.endswith('_3857.shp')
]
for filename_raw in tqdm(filenames_raw_polys, desc='Reproject shapefiles'):
filepath_raw = os.path.join(paths.DIR_DATA_TRAIN_MP_RAW, filename_raw)
filename_reproj = re.sub('.shp', '_3857.shp', filename_raw)
filepath_reproj = os.path.join(paths.DIR_DATA_TRAIN_MP_RAW,
filename_reproj)
if os.path.exists(filepath_reproj):
continue
command = 'ogr2ogr -t_srs EPSG:3857 {reproj} {raw}'.format(
reproj=filepath_reproj, raw=filepath_raw)
gdal_command_line.run_gdal_command(command, _logger)
def remove_feature_rasters_alpha_band() -> None:
filenames_raw = [
fn for fn in os.listdir(paths.DIR_DATA_TRAIN_FEATURES_RAW)
if fn.endswith('features.tif')
]
for filename_raw in tqdm(filenames_raw,
desc='Remove feature rasters alpha band'):
filepath_lock = os.path.join(paths.DIR_DATA_TRAIN_FEATURES_RAW,
filename_raw + '.lock')
filepath_raw = os.path.join(paths.DIR_DATA_TRAIN_FEATURES_RAW,
filename_raw)
filename_tmp = re.sub('features.tif', 'features_tmp.tif', filename_raw)
filepath_tmp = os.path.join(paths.DIR_DATA_TRAIN_FEATURES_RAW,
filename_tmp)
filepath_clean = os.path.join(paths.DIR_DATA_TRAIN_FEATURES_CLEAN,
filename_raw)
# Skip if output file already exists or if lock file exists
if os.path.exists(filepath_clean) or os.path.exists(filepath_lock):
continue
try:
file_lock = open(filepath_lock, 'x')
except OSError:
continue
try:
# Write in nodata values
command = 'gdal_calc.py -A {filepath_raw} --allBands=A -B {filepath_raw} --B_band=4 ' + \
'--outfile={filepath_tmp} --NoDataValue=-9999 --type=Int16 --co=COMPRESS=DEFLATE ' + \
'--co=TILED=YES --overwrite --calc="A * (B == 255) + -9999 * (B == 0)"'
command = command.format(filepath_raw=filepath_raw,
filepath_tmp=filepath_tmp)
gdal_command_line.run_gdal_command(command, _logger)
# Remove alpha band
options_removed = gdal.TranslateOptions(
bandList=[1, 2, 3],
outputType=gdal.GDT_Int16,
creationOptions=['COMPRESS=DEFLATE', 'TILED=YES'])
gdal.Translate(filepath_clean,
filepath_tmp,
options=options_removed)
except Exception as error_:
raise error_
finally:
if os.path.exists(filepath_tmp):
os.remove(filepath_tmp)
file_lock.close()
os.remove(filepath_lock)
def downsample_global_rasters() -> None:
_logger.info('Downsample rasters')
filenames = sorted([
fn for fn in os.listdir(paths.DIR_DATA_GLOBAL)
if re.search('L15-\d{4}E-\d{4}N.tif', fn)
])
for filename in tqdm(filenames, desc='Downsampling rasters'):
_logger.debug('Downsample raster: {}'.format(filename))
filepath_in = os.path.join(paths.DIR_DATA_GLOBAL, filename)
filepath_lock = filepath_in + '.lock'
if os.path.exists(filepath_lock) or not os.path.exists(
filepath_in) or os.path.exists(filepath_out):
continue
try:
file_lock = open(filepath_lock, 'x')
except OSError:
continue
try:
for pct in _DOWNSAMPLE_PCTS:
filename_out = re.sub('.tif', '_{}.tif'.format(pct), filename)
dir_out = os.path.join(
paths.DIR_DATA_GLOBAL,
paths.SUBDIR_DATA_TRAIN_DOWNSAMPLE.format(pct))
filepath_out = os.path.join(dir_out, filename_out)
if not os.path.exists(dir_out):
try:
os.makedirs(dir_out)
except OSError:
pass
command = 'gdal_translate -outsize {pct}% {pct}% -r bilinear {path_in} {path_out}'.format(
pct=pct, path_in=filepath_in, path_out=filepath_out)
gdal_command_line.run_gdal_command(command, _logger)
os.remove(filepath_in)
except Exception as error_:
raise error_
finally:
file_lock.close()
os.remove(filepath_lock)
def create_sampling_boundary_shapefiles() -> None:
raise AssertionError('This script has not been tested since being updated, be careful')
_logger.info('Creating sampling boundary shapefiles for original UQ training data')
_assert_encoding_assumptions_hold()
# Get list of completed responses rasters
tmp_filepath_reef_raster = os.path.join(paths.DIR_DATA_TRAIN_RAW, 'tmp_reef_only.tif')
tmp_filepath_reef_outline = os.path.join(paths.DIR_DATA_TRAIN_RAW, 'tmp_reef_outline.shp')
basename_reef_outline = os.path.splitext(os.path.basename(tmp_filepath_reef_outline))[0]
filepaths_responses = sorted([
os.path.join(paths.DIR_DATA_TRAIN_CLEAN, filename) for filename in os.listdir(paths.DIR_DATA_TRAIN_CLEAN)
if filename.endswith('_responses.tif')
])
for idx_responses, filepath_responses in enumerate(filepaths_responses):
_logger.debug('Creating boundaries for response file {} of {}'.format(idx_responses, len(filepaths_responses)))
filepath_boundary = re.sub('responses.tif', 'boundaries.shp', filepath_responses)
if os.path.exists(filepath_boundary):
_logger.debug('Boundary file already exists at: {}'.format(filepath_boundary))
continue
_logger.debug('Creating boundary file at: {}'.format(filepath_boundary))
# Get raster of only reef areas
_logger.debug('Creating reef-only raster')
min_reef_value = min(encodings.MAPPINGS[encodings.REEF_TOP], encodings.MAPPINGS[encodings.NOT_REEF_TOP])
command = 'gdal_calc.py -A {filepath_responses} --outfile={filepath_reef} --NoDataValue=-9999 ' + \
'--calc="1*(A>={min_value}) + -9999*(A<{min_value})"'
command = command.format(
filepath_responses=filepath_responses, filepath_reef=tmp_filepath_reef_raster, min_reef_value=min_reef_value)
gdal_command_line.run_gdal_command(command, _logger)
# Get shapefile of reef outline
_logger.debug('Creating reef outline shapefile')
command = 'gdal_polygonize.py {} {}'.format(tmp_filepath_reef_raster, tmp_filepath_reef_outline)
gdal_command_line.run_gdal_command(command, _logger)
# Get shapefile of sampling boundaries by buffering reef outline
_logger.debug('Creating buffered outline for boundary file')
command = 'ogr2ogr -f "ESRI Shapefile" {filepath_boundary} {filepath_outline} -dialect sqlite ' + \
'-sql "select ST_buffer(geometry, 64) as geometry from {basename_outline}"'
command = command.format(
filepath_boundary=filepath_boundary, filepath_outline=tmp_filepath_reef_outline,
basename_outline=basename_reef_outline
)
gdal_command_line.run_gdal_command(command, _logger)
# Clean up
_logger.debug('Remove temporary files')
os.remove(tmp_filepath_reef_raster)
for filename_outline in os.listdir(paths.DIR_DATA_TRAIN_RAW):
if not re.search(basename_reef_outline, filename_outline):
continue
os.remove(os.path.join(paths.DIR_DATA_TRAIN_RAW, filename_outline))
def create_sampling_boundary_shapefiles() -> None:
_logger.info('Creating sampling boundary shapefiles for supplemental training data')
# Get list of supplemental shapefiles
tmp_filepath_raster = os.path.join(paths.DIR_DATA_TRAIN_RAW, 'tmp_raster_valid.tif')
tmp_filepath_polygon = os.path.join(paths.DIR_DATA_TRAIN_RAW, 'tmp_outline_valid.shp')
basename_polygon = os.path.splitext(os.path.basename(tmp_filepath_polygon))[0]
filepaths_shapes = sorted([
os.path.join(paths.DIR_DATA_TRAIN_CLEAN, filename) for filename in os.listdir(paths.DIR_DATA_TRAIN_CLEAN)
if filename.endswith('_model_class.shp')
])
for filepath_shape in tqdm(filepaths_shapes, desc='Create sampling boundary shapefiles'):
filepath_boundary = re.sub('.shp', 'boundaries.shp', filepath_shape)
if os.path.exists(filepath_boundary):
_logger.debug('Boundary file already exists at: {}'.format(filepath_boundary))
continue
_logger.debug('Creating boundary file at: {}'.format(filepath_boundary))
_logger.debug('Creating valid sampling area raster')
command = 'gdal_rasterize -burn 1 -tr 5 5 -a_nodata -9999 -init -9999 ' + \
'{filepath_shape} {tmp_filepath_raster}'
command = command.format(filepath_shape=filepath_shape, tmp_filepath_raster=tmp_filepath_raster)
gdal_command_line.run_gdal_command(command, _logger)
_logger.debug('Creating valid sampling area outline')
command = 'gdal_polygonize.py {} {}'.format(tmp_filepath_raster, tmp_filepath_polygon)
gdal_command_line.run_gdal_command(command, _logger)
_logger.debug('Creating buffered outline for boundary file')
command = 'ogr2ogr -f "ESRI Shapefile" {filepath_boundary} {tmp_filepath_polygon} -dialect sqlite ' + \
'-sql "select ST_buffer(geometry, 64) as geometry from {basename_polygon}"'
command = command.format(
filepath_boundary=filepath_boundary, tmp_filepath_polygon=tmp_filepath_polygon,
basename_polygon=basename_polygon
)
gdal_command_line.run_gdal_command(command, _logger)
_logger.debug('Remove temporary files')
os.remove(tmp_filepath_raster)
for extension in ('.dbf', '.prj', '.shp', '.shx'):
os.remove(os.path.join(paths.DIR_DATA_TRAIN_RAW, basename_polygon + extension))
def create_sampling_boundary_shapefiles() -> None:
_logger.info(
'Creating sampling boundary shapefiles for Millennium Project training data'
)
filenames_responses = sorted([
filename for filename in os.listdir(paths.DIR_DATA_TRAIN_MP_CLEAN)
if filename.endswith('_responses_custom.tif')
])
desc = 'Create sampling boundary shapefiles'
for idx_filepath, filename_response in enumerate(
tqdm(filenames_responses, desc=desc)):
filepath_response = os.path.join(paths.DIR_DATA_TRAIN_MP_CLEAN,
filename_response)
filename_sqlvalid = re.sub('-', '_',
os.path.basename(filepath_response))
tmp_filename_raster = re.sub('_responses_custom.tif',
'_tmp_noland.tif', filename_sqlvalid)
tmp_filepath_raster = os.path.join(paths.DIR_DATA_TRAIN_MP_BOUNDS,
tmp_filename_raster)
tmp_filename_outline = re.sub('_responses_custom.tif',
'_tmp_noland.shp', filename_sqlvalid)
tmp_filepath_outline = os.path.join(paths.DIR_DATA_TRAIN_MP_BOUNDS,
tmp_filename_outline)
basename_outline = os.path.splitext(
os.path.basename(tmp_filepath_outline))[0]
filename_boundary = re.sub('_responses_custom.tif', '_boundaries.shp',
filename_response)
filepath_boundary = os.path.join(paths.DIR_DATA_TRAIN_MP_BOUNDS,
filename_boundary)
filepath_lock = filepath_boundary + '.lock'
if os.path.exists(filepath_boundary):
_logger.debug('Boundary file already exists at: {}'.format(
filepath_boundary))
continue
if os.path.exists(filepath_lock):
_logger.debug('Boundary file already being processed: {}'.format(
filepath_boundary))
continue
try:
file_lock = open(filepath_lock, 'x')
except OSError:
continue
_logger.debug('Creating boundary file {} ({} total): {}'.format(
idx_filepath, len(filenames_responses), filepath_response))
try:
# Get raster of areas we care about, those that are not land
_logger.debug('Creating non-land raster')
code_land = encodings_mp.CODE_LAND
command = 'gdal_calc.py -A {filepath_response} --outfile={tmp_raster} --NoDataValue=-9999 ' + \
'--calc="1*(A!={code_land}) + -9999*(A=={code_land})"'
command = command.format(filepath_response=filepath_response,
tmp_raster=tmp_filepath_raster,
code_land=code_land)
gdal_command_line.run_gdal_command(command, _logger)
# Get shapefile for outline of areas we care about
_logger.debug('Creating non-land outline shapefile')
command = 'gdal_polygonize.py {} {}'.format(
tmp_filepath_raster, tmp_filepath_outline)
gdal_command_line.run_gdal_command(command, _logger)
# Get shapefile of sampling boundaries by buffering reef outline
_logger.debug('Creating buffered outline for boundary file')
command = 'ogr2ogr -f "ESRI Shapefile" {filepath_boundary} {tmp_outline} -dialect sqlite ' + \
'-sql "select ST_buffer(geometry, 64) as geometry from {basename_outline}"'
command = command.format(filepath_boundary=filepath_boundary,
tmp_outline=tmp_filepath_outline,
basename_outline=basename_outline)
gdal_command_line.run_gdal_command(command, _logger)
except Exception as error_:
raise error_
finally:
if os.path.exists(tmp_filepath_raster):
os.remove(tmp_filepath_raster)
for filename in os.listdir(os.path.dirname(tmp_filepath_outline)):
if not re.search(basename_outline, filename):
continue
os.remove(
os.path.join(os.path.dirname(tmp_filepath_outline),
filename))
file_lock.close()
os.remove(filepath_lock)
def create_millennium_project_quad_rasters_custom_classes() -> None:
_logger.info('Create Millennium Project response quad rasters')
if not os.path.exists(paths.DIR_DATA_TRAIN_MP_CLEAN):
os.makedirs(paths.DIR_DATA_TRAIN_MP_CLEAN)
filenames_polys = [
filename for filename in os.listdir(paths.DIR_DATA_TRAIN_MP_RAW)
if filename.startswith('L15-') and filename.endswith('responses.shp')
]
missing_features = list()
for idx_filename, filename_poly in enumerate(
tqdm(filenames_polys, desc='Create Millennium Project rasters')):
_logger.debug('Create raster {} ({} total): {}'.format(
idx_filename, len(filenames_polys), filename_poly))
# Set filepaths
filepath_src = os.path.join(paths.DIR_DATA_TRAIN_MP_RAW, filename_poly)
filename_custom = re.sub('responses.shp', 'responses_tmp.shp',
filename_poly)
filepath_custom = os.path.join(paths.DIR_DATA_TRAIN_MP_RAW,
filename_custom)
filename_dest = re.sub('.shp', '_custom.tif', filename_poly)
filepath_dest = os.path.join(paths.DIR_DATA_TRAIN_MP_CLEAN,
filename_dest)
filepath_lock = filepath_dest + '.lock'
if os.path.exists(filepath_dest) or os.path.exists(filepath_lock):
continue
try:
file_lock = open(filepath_lock, 'x')
except OSError:
continue
try:
# Try to find existing features file, may be either raw or clean, but also may not be available from Vulcan
filename_features = re.search(
r'L15-\d{4}E-\d{4}N', filename_poly).group() + '_features.tif'
filepath_features = os.path.join(paths.DIR_DATA_TRAIN_FEATURES,
filename_features)
if not os.path.exists(filepath_features):
filepath_features = os.path.join(
paths.DIR_DATA_TRAIN_FEATURES_CLEAN, filename_features)
if not os.path.exists(filepath_features):
_logger.warning(
'Features file not available in raw or clean dirs: {}'.
format(filename_features))
missing_features.append(filename_features)
continue
# Create shapefile with custom classes
crs = fiona.crs.from_epsg(SHP_EPSG)
with fiona.open(filepath_custom,
'w',
driver=SHP_DRIVER,
crs=crs,
schema=SHP_SCHEMA) as file_custom:
for original_feature in fiona.open(filepath_src):
original_code = original_feature['properties']['L4_CODE']
custom_code = encodings_mp.MAPPINGS_CUSTOM[original_code]
custom_feature = {
'properties': {
'custom': custom_code
},
'geometry': original_feature['geometry']
}
file_custom.write(custom_feature)
# Get rasterize parameters
raster_features = gdal.Open(filepath_features)
cols = raster_features.RasterXSize
rows = raster_features.RasterYSize
llx, xres, _, y0, _, yres = raster_features.GetGeoTransform()
urx = llx + cols * xres
y1 = y0 + rows * yres
lly = min([y0, y1])
ury = max([y0, y1])
# Rasterize
command = 'gdal_rasterize -ot Int16 -co COMPRESS=DEFLATE -co TILED=YES -a_nodata -9999 -init -9999 ' \
'-a custom -te {llx} {lly} {urx} {ury} -tr {xres} {yres} {filepath_custom} {filepath_dest}'
command = command.format(llx=llx,
lly=lly,
urx=urx,
ury=ury,
xres=xres,
yres=yres,
filepath_custom=filepath_custom,
filepath_dest=filepath_dest)
gdal_command_line.run_gdal_command(command, _logger)
except Exception as error_:
raise error_
finally:
for filename in os.listdir(os.path.dirname(filepath_custom)):
if re.search(os.path.splitext(filename_custom)[0], filename):
os.remove(
os.path.join(os.path.dirname(filepath_custom),
filename))
file_lock.close()
os.remove(filepath_lock)
assert not missing_features, 'Missing features files: {}'.format(
', '.join(missing_features))
def rasterize_response_quads() -> None:
raise AssertionError(
'This script has not been tested since being updated, be careful')
_assert_encoding_assumptions_hold()
filenames = [
filename for filename in os.listdir(paths.DIR_DATA_TRAIN_RAW)
if filename.endswith('.shp')
]
for idx_filename, filename in enumerate(filenames):
print('\rRasterizing shapefile {} of {}'.format(
1 + idx_filename, len(filenames)))
# Get quad and filepaths
quad = re.search(r'L15-\d{4}E-\d{4}N', filename).group()
filepath_features = os.path.join(
paths.DIR_DATA_TRAIN_CLEAN,
quad + data_bucket.FILENAME_SUFFIX_FEATURES)
filepath_source_responses = os.path.join(paths.DIR_DATA_TRAIN_RAW,
filename)
filepath_dest_lwr = re.sub('features.tif', 'responses_lwr.tif',
filepath_features)
filepath_dest_lwrn = re.sub('features.tif', 'responses_lwrn.tif',
filepath_features)
assert os.path.exists(
filepath_features
), 'Download all feature quads before rasterizing response quads'
# Get rasterize parameters
raster_features = gdal.Open(filepath_features)
srs = osr.SpatialReference(wkt=raster_features.GetProjection())
cols = raster_features.RasterXSize
rows = raster_features.RasterYSize
llx, xres, _, y0, _, yres = raster_features.GetGeoTransform()
urx = llx + cols * xres
y1 = y0 + rows * yres
lly = min([y0, y1])
ury = max([y0, y1])
# Rasterize into Land-Water-Reef-NotReef
options_rasterize = gdal.RasterizeOptions(
outputType=gdal.GDT_Int16,
creationOptions=['COMPRESS=DEFLATE', 'TILED=YES'],
outputBounds=[llx, lly, urx, ury],
outputSRS=srs,
xRes=xres,
yRes=yres,
noData=-9999,
initValues=-9999,
attribute='class_code')
raster_out = gdal.Rasterize(filepath_dest_lwrn,
filepath_source_responses,
options=options_rasterize)
del raster_out
# Remove cloud-shade and unknown classes. Unknown classes could be anything from water to reef to clouds, while
# cloud-shade is not reliable as the map was created with the analytical mosaic rather than the visual mosaic.
min_nodata = min(encodings.MAPPINGS[encodings.CLOUD_SHADE],
encodings.MAPPINGS[encodings.UNKNOWN])
command = 'gdal_calc.py -A {filepath} --outfile {filepath} --NoDataValue=-9999 --overwrite ' + \
'--calc="A * (A < {min_nodata}) + -9999 * (A >= {min_nodata})"'
command = command.format(filepath=filepath_dest_lwrn,
min_nodata=min_nodata)
gdal_command_line.run_gdal_command(command, _logger)
# Create Land-Water-Reef
val_reef = encodings.MAPPINGS[encodings.REEF_TOP]
val_notreef = encodings.MAPPINGS[encodings.NOT_REEF_TOP]
command = 'gdal_calc.py -A {filepath_lwrn} --outfile={filepath_lwr} --NoDataValue=-9999 --overwrite ' + \
'--calc="A * (A != {val_notreef}) + {val_reef} * (A == {val_notreef})"'
command = command.format(filepath_lwrn=filepath_dest_lwrn,
filepath_lwr=filepath_dest_lwr,
val_notreef=val_notreef,
val_reef=val_reef)
gdal_command_line.run_gdal_command(command, _logger)
]
for fn in fns:
quad = re.search('L15-\d{4}E-\d{4}N', fn).group()
dest = os.path.join('../../responses_mp_supp', quad + '_water_cloud.tif')
if not os.path.exists(fn):
continue
shutil.copy(fn, dest)
# Convert land to 2000
fns = [fn for fn in os.listdir() if fn.endswith('_land.tif')]
for fn in fns:
dest = os.path.join('../clean', os.path.basename(fn))
command = 'gdal_calc.py -A {fn} --outfile={dest} --calc="2000*(A==1)+-9999*(A!=1)" --NoDataValue=-9999 --co=TILED=YES --co=COMPRESS=DEFLATE'.format(
fn=fn, dest=dest)
gdal_command_line.run_gdal_command(command)
# Convert water to 2001
fns = [fn for fn in os.listdir() if fn.endswith('_water.tif')]
for fn in fns:
dest = os.path.join('../clean', os.path.basename(fn))
command = 'gdal_calc.py -A {fn} --outfile={dest} --calc="2001*(A==2)+-9999*(A!=2)" --NoDataValue=-9999 --co=TILED=YES --co=COMPRESS=DEFLATE'.format(
fn=fn, dest=dest)
gdal_command_line.run_gdal_command(command)
# Convert clouds to 2002
fns = [fn for fn in os.listdir() if fn.endswith('_water_cloud.tif')]
for fn in fns:
dest = os.path.join('../clean', os.path.basename(fn))
command = 'gdal_calc.py -A {fn} --outfile={dest} --calc="2001*(A==2)+2002*(A==23)+-9999*numpy.logical_and(A!=2, A!=23)" --NoDataValue=-9999 --co=TILED=YES --co=COMPRESS=DEFLATE'.format(
fn=fn, dest=dest)