def _upload_vrt(key: str, vrt: str, prefix: str) -> Dict[str, Any]:
bucket = utils.get_bucket()
key = os.path.join(prefix, key, vrt)
LOGGER.info(f"Upload vrt to {bucket} {key}")
return S3.upload_file(vrt, bucket, key)
def upload_geojsons(
processed_tiles: List[Tile],
existing_tiles: List[Tile],
prefix: str,
bucket: str = utils.get_bucket(),
ignore_existing_tiles=False,
) -> List[Dict[str, Any]]:
"""Create geojson listing all tiles and upload to S3."""
response: List[Dict[str, Any]] = list()
if ignore_existing_tiles:
all_tiles = processed_tiles
else:
all_tiles = processed_tiles + existing_tiles
# Upload a tiles.geojson for the default format even if no tiles
dst_formats: Set[DstFormat] = {DstFormat(GLOBALS.default_dst_format)}
for tile in all_tiles:
for fmt in tile.dst.keys():
dst_formats.add(DstFormat(fmt))
for dst_format in dst_formats:
fc: FeatureCollection = generate_feature_collection(
all_tiles, dst_format)
response.append(
_upload_extent(fc, prefix=prefix, dst_format=dst_format))
key = os.path.join(prefix, dst_format, "tiles.geojson")
response.append(_upload_geojson(fc, bucket, key))
return response
def upload(self) -> None:
try:
bucket = utils.get_bucket()
for dst_format in self.local_dst.keys():
local_tiff_path = self.local_dst[dst_format].uri
LOGGER.info(f"Upload {local_tiff_path} to s3")
_ = upload_s3(
local_tiff_path,
bucket,
self.dst[dst_format].uri,
)
# Also upload the stats sidecar file that gdalinfo creates
# Use the default format for path because we only create 1 sidecar
local_stats_path = self.local_dst[
self.default_format].uri + stats_ext
if os.path.isfile(local_stats_path):
LOGGER.info(f"Upload {local_stats_path} to s3")
_ = upload_s3(
local_stats_path,
bucket,
self.dst[dst_format].uri + stats_ext,
)
except Exception as e:
LOGGER.error(f"Could not upload file {self.tile_id}")
LOGGER.exception(str(e))
self.status = "failed"
def _upload_extent(
fc: FeatureCollection,
prefix: str,
dst_format: str,
bucket: str = utils.get_bucket(),
) -> Dict[str, Any]:
"""Create geojson file for tileset extent and upload to S3."""
extent_fc = _union_tile_geoms(fc)
key = os.path.join(prefix, dst_format, "extent.geojson")
return _upload_geojson(extent_fc, bucket, key)
def create_geojsons(
resources: List[Tuple[str, str, str]],
dataset: str,
version: str,
prefix: str,
merge_existing: bool,
) -> None:
get_files = {"s3": get_aws_files, "gs": get_gs_files}
tiles: List[DummyTile] = list()
for provider, bucket, key in resources:
files = get_files[provider](bucket, key)
for uri in files:
src = RasterSource(uri)
tiles.append(DummyTile({"geotiff": src}))
data_lake_bucket = get_bucket()
target_prefix = f"{dataset}/{version}/{prefix.strip('/')}/"
# Don't bother checking for existing tiles unless we're going to use them
existing_tiles = list()
if merge_existing:
existing_uris = get_aws_files(data_lake_bucket, target_prefix)
for uri in existing_uris:
src = RasterSource(uri)
existing_tiles.append(DummyTile({"geotiff": src}))
upload_geometries.upload_geojsons(
tiles, # type: ignore
existing_tiles, # type: ignore
bucket=data_lake_bucket,
prefix=target_prefix,
ignore_existing_tiles=not merge_existing,
)
def bucket(self):
return get_bucket()