def prep_intensity(tiles, **kwargs):
"""
Reclassify our final year/date raster to 0 | 55
this will then be resampled at several levels to get our intensity input
:param tiles:
:return:
"""
root = kwargs["root"]
name = kwargs["name"]
max_ras_value = 55
for tile in tiles:
f_name, year, folder, tile_id = file_details(tile)
output = output_file(root, "tiles", tile_id, name, year, "source_intensity.tif")
cmd = ["reclass", tile, output, str(max_ras_value)]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Prepared intensity for file: " + tile)
yield output
else:
logging.error("Failed to prepare intensity for file: " + tile)
logging.error(e)
raise sp.CalledProcessError
def encode_date_conf(tiles, **kwargs):
"""
:param tiles:
:param kwargs:
:return:
"""
root = kwargs["root"]
name = kwargs["name"]
for tile in tiles:
f_name, year, folder, tile_id = file_details(tile)
output = output_file(root, "tiles", tile_id, name, year, f_name)
cmd = ["encode_date_conf.py", "-i", tile, "-o", output, "-y", str(year)]
if f_name == "day.tif":
cmd += ["-m", "date"]
else:
cmd += ["-m", "conf"]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Encoded file: " + tile)
yield output
else:
logging.error("Failed to encode file: " + tile)
logging.error(e)
raise sp.CalledProcessError
def upload_rgb_wm_s3(tiles, **kwargs):
env = kwargs["env"]
path = kwargs["paths"]["resampled_rgb"]
for tile in tiles:
if env == "test":
logging.info(
"Test run, skipped upload preprocessed tiles to S3: " + tile)
yield tile
else:
f_name, zoom, folder, tile_id = file_details(tile)
output = path.format(env=env, zoom=zoom, tile_id=tile_id)
cmd = ["aws", "s3", "cp", tile, output]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Upload file to " + output)
yield tile
else:
logging.error("Failed to upload file to " + output)
logging.error(e)
raise sp.CalledProcessError
def get_dataframe(tile_pairs):
for tile_pair in tile_pairs:
tile = tile_pair[0]
emissions = tile_pair[1]
climate_mask = tile_pair[2]
f_name, year, folder, tile_id = file_details(tile)
tiles = [tile_pair[0]]
if emissions:
tiles = tiles + [emissions]
if climate_mask:
tiles = tiles + [climate_mask]
try:
logging.info("Extract points for tile: " + str(tiles))
df = raster2df(*tiles, calc_area=True)
except Exception as e:
logging.error("Failed to extract points for tiles: " + str(tiles))
logging.error(e)
raise e
else:
if not emissions:
df["val1"] = 0
if not climate_mask:
df["val2"] = 0
yield year, tile_id, df
def merge_years(tile_dicts, **kwargs):
root = kwargs["root"]
name = kwargs["name"]
for tile_dict in tile_dicts:
logging.info(str(tile_dict))
f_name, year, folder, tile_id = file_details(
list(tile_dict.values())[0])
year_str = preprocessed_years_str(sorted(tile_dict.keys()))
output = output_file(root, "tiles", tile_id, name, year_str,
"day_conf.tif")
input = sort_dict(tile_dict)
cmd = ["combine{}".format(len(input))] + input + [output]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Combined files: " + str(input))
yield output
else:
logging.error("Failed to combine files: " + str(input))
logging.error(e)
raise sp.CalledProcessError
def change_pixel_depth(tiles, **kwargs):
try:
root = kwargs["root"]
name = kwargs["name"]
except KeyError:
logging.warning("Wrong number of arguments")
else:
for tile in tiles:
f_name, year, folder, tile_id = file_details(tile)
output = output_file(root, "tiles", tile_id, name, year, f_name)
min_x, min_y, max_x, max_y = get_bbox_by_tile_id(tile_id)
cmd = [
"gdalwarp",
"-ot",
"UInt16",
"-dstnodata",
"0",
"-co",
"COMPRESS=LZW",
"-co",
"TILED=YES",
"-co",
"SPARSE_OK=TRUE",
"-r",
"near",
"-te",
str(min_x),
str(min_y),
str(max_x),
str(max_y),
"-ts",
str(TILE_WIDTH),
str(TILE_WIDTH),
tile,
output,
]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Changed pixel depth for file: " + tile)
yield output
else:
logging.error("Failed to change pixel depth for file: " + tile)
logging.error(e)
raise sp.CalledProcessError
def upload_raw_tile_s3(tiles, **kwargs):
"""
Backups raw day and conf tiles for single years
:param tiles:
:param kwargs:
:return:
"""
# TODO: This function returns the input and should be marked accordingly
env = kwargs["env"]
path = kwargs["paths"]["raw_backup"]
for tile in tiles:
if env == "test":
logging.info(
"Test run, skipped upload preprocessed tiles to S3: " + tile)
yield tile
else:
f_name, year, folder, tile_id = file_details(tile)
output = path.format(env=env,
year=year,
product=f_name[:-4],
tile_id=tile_id)
cmd = ["aws", "s3", "cp", tile, output]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Upload file " + output)
yield tile
else:
logging.error("Failed to upload file to " + output)
logging.error(e)
raise sp.CalledProcessError
def combine_date_conf_pairs(pairs, **kwargs):
root = kwargs["root"]
name = kwargs["name"]
for pair in pairs:
f_name, year, folder, tile_id = file_details(pair["day"])
output = output_file(root, "tiles", tile_id, name, year,
"day_conf.tif")
cmd = ["add2", pair["day"], pair["conf"], output]
logging.debug(cmd)
p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
o, e = p.communicate()
logging.debug(o)
if p.returncode == 0:
logging.info("Combined files into: " + output)
yield output
else:
logging.error("Failed to combine files into: " + output)
logging.error(e)
raise sp.CalledProcessError
def validate_tiles(tiles, **kwargs):
for tile in tiles:
f_name, year, folder, tile_id = file_details(tile)
if f_name == "day.tif":
expected_range = range(0, 367)
else:
expected_range = [0, 2, 3]
gtif = gdal.Open(tile)
srcband = gtif.GetRasterBand(1)
stats = srcband.GetStatistics(True, True)
min = stats[0]
max = stats[1]
if min not in expected_range or max not in expected_range:
raise ValueError(
f"Tile f{tile_id}, year {year}, file {f_name} "
f"had min, max of {min, max}, which is outside of the expected range"
)
yield tile