def prep_intensity(tiles, **kwargs): """ Reclassify our final year/date raster to 0 | 55 this will then be resampled at several levels to get our intensity input :param tiles: :return: """ root = kwargs["root"] name = kwargs["name"] max_ras_value = 55 for tile in tiles: f_name, year, folder, tile_id = file_details(tile) output = output_file(root, "tiles", tile_id, name, year, "source_intensity.tif") cmd = ["reclass", tile, output, str(max_ras_value)] logging.debug(cmd) p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) o, e = p.communicate() logging.debug(o) if p.returncode == 0: logging.info("Prepared intensity for file: " + tile) yield output else: logging.error("Failed to prepare intensity for file: " + tile) logging.error(e) raise sp.CalledProcessError
def encode_date_conf(tiles, **kwargs): """ :param tiles: :param kwargs: :return: """ root = kwargs["root"] name = kwargs["name"] for tile in tiles: f_name, year, folder, tile_id = file_details(tile) output = output_file(root, "tiles", tile_id, name, year, f_name) cmd = ["encode_date_conf.py", "-i", tile, "-o", output, "-y", str(year)] if f_name == "day.tif": cmd += ["-m", "date"] else: cmd += ["-m", "conf"] logging.debug(cmd) p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) o, e = p.communicate() logging.debug(o) if p.returncode == 0: logging.info("Encoded file: " + tile) yield output else: logging.error("Failed to encode file: " + tile) logging.error(e) raise sp.CalledProcessError
def upload_rgb_wm_s3(tiles, **kwargs): env = kwargs["env"] path = kwargs["paths"]["resampled_rgb"] for tile in tiles: if env == "test": logging.info( "Test run, skipped upload preprocessed tiles to S3: " + tile) yield tile else: f_name, zoom, folder, tile_id = file_details(tile) output = path.format(env=env, zoom=zoom, tile_id=tile_id) cmd = ["aws", "s3", "cp", tile, output] logging.debug(cmd) p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) o, e = p.communicate() logging.debug(o) if p.returncode == 0: logging.info("Upload file to " + output) yield tile else: logging.error("Failed to upload file to " + output) logging.error(e) raise sp.CalledProcessError
def get_dataframe(tile_pairs): for tile_pair in tile_pairs: tile = tile_pair[0] emissions = tile_pair[1] climate_mask = tile_pair[2] f_name, year, folder, tile_id = file_details(tile) tiles = [tile_pair[0]] if emissions: tiles = tiles + [emissions] if climate_mask: tiles = tiles + [climate_mask] try: logging.info("Extract points for tile: " + str(tiles)) df = raster2df(*tiles, calc_area=True) except Exception as e: logging.error("Failed to extract points for tiles: " + str(tiles)) logging.error(e) raise e else: if not emissions: df["val1"] = 0 if not climate_mask: df["val2"] = 0 yield year, tile_id, df
def merge_years(tile_dicts, **kwargs): root = kwargs["root"] name = kwargs["name"] for tile_dict in tile_dicts: logging.info(str(tile_dict)) f_name, year, folder, tile_id = file_details( list(tile_dict.values())[0]) year_str = preprocessed_years_str(sorted(tile_dict.keys())) output = output_file(root, "tiles", tile_id, name, year_str, "day_conf.tif") input = sort_dict(tile_dict) cmd = ["combine{}".format(len(input))] + input + [output] logging.debug(cmd) p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) o, e = p.communicate() logging.debug(o) if p.returncode == 0: logging.info("Combined files: " + str(input)) yield output else: logging.error("Failed to combine files: " + str(input)) logging.error(e) raise sp.CalledProcessError
def change_pixel_depth(tiles, **kwargs): try: root = kwargs["root"] name = kwargs["name"] except KeyError: logging.warning("Wrong number of arguments") else: for tile in tiles: f_name, year, folder, tile_id = file_details(tile) output = output_file(root, "tiles", tile_id, name, year, f_name) min_x, min_y, max_x, max_y = get_bbox_by_tile_id(tile_id) cmd = [ "gdalwarp", "-ot", "UInt16", "-dstnodata", "0", "-co", "COMPRESS=LZW", "-co", "TILED=YES", "-co", "SPARSE_OK=TRUE", "-r", "near", "-te", str(min_x), str(min_y), str(max_x), str(max_y), "-ts", str(TILE_WIDTH), str(TILE_WIDTH), tile, output, ] logging.debug(cmd) p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) o, e = p.communicate() logging.debug(o) if p.returncode == 0: logging.info("Changed pixel depth for file: " + tile) yield output else: logging.error("Failed to change pixel depth for file: " + tile) logging.error(e) raise sp.CalledProcessError
def upload_raw_tile_s3(tiles, **kwargs): """ Backups raw day and conf tiles for single years :param tiles: :param kwargs: :return: """ # TODO: This function returns the input and should be marked accordingly env = kwargs["env"] path = kwargs["paths"]["raw_backup"] for tile in tiles: if env == "test": logging.info( "Test run, skipped upload preprocessed tiles to S3: " + tile) yield tile else: f_name, year, folder, tile_id = file_details(tile) output = path.format(env=env, year=year, product=f_name[:-4], tile_id=tile_id) cmd = ["aws", "s3", "cp", tile, output] logging.debug(cmd) p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) o, e = p.communicate() logging.debug(o) if p.returncode == 0: logging.info("Upload file " + output) yield tile else: logging.error("Failed to upload file to " + output) logging.error(e) raise sp.CalledProcessError
def combine_date_conf_pairs(pairs, **kwargs): root = kwargs["root"] name = kwargs["name"] for pair in pairs: f_name, year, folder, tile_id = file_details(pair["day"]) output = output_file(root, "tiles", tile_id, name, year, "day_conf.tif") cmd = ["add2", pair["day"], pair["conf"], output] logging.debug(cmd) p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) o, e = p.communicate() logging.debug(o) if p.returncode == 0: logging.info("Combined files into: " + output) yield output else: logging.error("Failed to combine files into: " + output) logging.error(e) raise sp.CalledProcessError
def validate_tiles(tiles, **kwargs): for tile in tiles: f_name, year, folder, tile_id = file_details(tile) if f_name == "day.tif": expected_range = range(0, 367) else: expected_range = [0, 2, 3] gtif = gdal.Open(tile) srcband = gtif.GetRasterBand(1) stats = srcband.GetStatistics(True, True) min = stats[0] max = stats[1] if min not in expected_range or max not in expected_range: raise ValueError( f"Tile f{tile_id}, year {year}, file {f_name} " f"had min, max of {min, max}, which is outside of the expected range" ) yield tile