def collect_rgb_tile_ids(zoom_tiles):
tile_list = None
for tile in zoom_tiles:
if tile[0] == 12:
tile_list = tile[1]
break
tile_ids = [get_tile_id(tile) for tile in tile_list]
return tile_ids
def resample(tiles, **kwargs):
root = kwargs["root"]
name = kwargs["name"]
resample_method = kwargs["resample_method"]
zoom = kwargs["zoom"]
for tile in tiles:
tile_id = get_tile_id(tile)
output = output_file(
root,
"tiles",
tile_id,
"resample",
"zoom_{}".format(zoom),
"{}.tif".format(name),
)
cell_size = str(ras_util.get_cell_size(zoom, "degrees"))
# mem_pct = ras_util.get_mem_pct()
cmd = [
"gdal_translate",
tile,
output,
"-co",
"COMPRESS=DEFLATE",
"-r",
resample_method,
"-tr",
cell_size,
cell_size,
"-co",
"TILED=YES",
]
# TODO: figure out how to best manage memory
# cmd += ['--config', 'GDAL_CACHEMAX', mem_pct]
try:
logging.debug(cmd)
sp.check_call(cmd)
except sp.CalledProcessError as e:
logging.error("Failed to resample file: " + tile)
logging.error(e)
raise sp.CalledProcessError
else:
logging.info("Resampled file: " + tile)
yield output
def match_emissions(tiles, **kwargs):
root = kwargs["root"]
name = kwargs["name"]
for tile in tiles:
tile_id = get_tile_id(tile)
path = PurePath(root, "climate", name, tile_id + ".tif")
if PosixPath(path).exists():
logging.info("Found matching emission file: " + path.as_posix())
yield tile, path.as_posix()
else:
logging.warning("Could not find matching emission file: " +
path.as_posix())
yield tile, None
def match_climate_mask(tile_pairs, **kwargs):
root = kwargs["root"]
name = kwargs["name"]
for tile_pair in tile_pairs:
tile_id = get_tile_id(tile_pair[0])
path = PurePath(root, "climate", name, tile_id + ".tif")
if PosixPath(path).exists():
logging.info("Found matching climate mask: " + path.as_posix())
yield tile_pair[0], tile_pair[1], path.as_posix()
else:
# Not all tiles have a corresponding climate mask
logging.warning("Could not find matching climate mask: " +
path.as_posix())
yield tile_pair[0], tile_pair[1], None
def upload_day_conf_s3_gfw_pro(tiles, pro_tiles, **kwargs):
"""
Uploads final encode tiles which include all years to S3 GFW-Pro account
:param tiles:
:param kwargs:
:return:
"""
# TODO: This function returns the input and should be marked accordingly
env = kwargs["env"]
path = kwargs["paths"]["pro"]
for tile in tiles:
if env != "prod": # No Staging environment for Pro!
logging.info(
"Test run, skipped upload preprocessed tiles to S3: " + tile)
yield tile
else:
tile_id = get_tile_id(tile)
if tile_id in pro_tiles.keys():
output = path.format(pro_id=pro_tiles[tile_id])
cmd = [
"aws",
"s3",
"cp",
tile,
output,
"--profile",
"GFWPro_gfwpro-raster-data_remote",
]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Uploaded file to GFW Pro: " + output)
else:
logging.error("Failed to upload file to GFW Pro: " +
output)
logging.error(e)
raise sp.CalledProcessError
yield tile
def upload_day_conf_s3_archive(tiles, **kwargs):
"""
Uploads final encode tiles which include all years to archive
:param tiles:
:param kwargs:
:return:
"""
# TODO: This function returns the input and should be marked accordingly
env = kwargs["env"]
preprocessed_years = list(kwargs["preprocessed_years"])
s3_url = "s3://gfw2-data/forest_change/umd_landsat_alerts/archive/pipeline/tiles/{}/day_conf/{}/day_conf.tif"
preprocessed_years.append(max(preprocessed_years) + 1)
year_str = preprocessed_years_str(preprocessed_years)
for tile in tiles:
tile_id = get_tile_id(tile)
output = s3_url.format(tile_id, year_str)
if env == "test":
logging.info("Test run, skipped upload preprocessed tile " + tile +
" to " + output)
yield tile
else:
cmd = ["aws", "s3", "cp", tile, output]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Uploaded file " + output)
yield tile
else:
logging.error("Failed to upload file to " + output)
logging.error(e)
raise sp.CalledProcessError
def collect_resampled_tiles(root):
resampled_tiles = dict()
for tile in glob.iglob(root + "/tiles/**/resample/*/*.tif",
recursive=True):
tile_id = get_tile_id(tile)
zoom_level = PurePath(tile).parts[-2]
if (zoom_level, tile_id) not in resampled_tiles.keys():
resampled_tiles[(zoom_level, tile_id)] = list()
resampled_tiles[(zoom_level, tile_id)].append(tile)
if len(resampled_tiles[(zoom_level, tile_id)]) == 2:
logging.info("Created pairs for: " + str((zoom_level, tile_id)))
yield sorted(resampled_tiles[(zoom_level, tile_id)])
for key, value in resampled_tiles.items():
if len(value) < 2:
logging.warning("Could not create pair for: " + str(key))
def upload_day_conf_s3(tiles, **kwargs):
"""
Uploads final encode tiles which include all years
:param tiles:
:param kwargs:
:return:
"""
# TODO: This function returns the input and should be marked accordingly
env = kwargs["env"]
path = kwargs["paths"]["analysis"]
for tile in tiles:
if env == "test":
logging.info(
"Test run, skipped upload preprocessed tiles to S3: " + tile)
yield tile
else:
tile_id = get_tile_id(tile)
output = path.format(env=env, tile_id=tile_id)
cmd = ["aws", "s3", "cp", tile, output]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Uploaded file " + output)
yield tile
else:
logging.error("Failed to upload file to " + output)
logging.error(e)
raise sp.CalledProcessError