def _upload_vrt(key: str, vrt: str, prefix: str) -> Dict[str, Any]: bucket = utils.get_bucket() key = os.path.join(prefix, key, vrt) LOGGER.info(f"Upload vrt to {bucket} {key}") return S3.upload_file(vrt, bucket, key)
def upload_geojsons( processed_tiles: List[Tile], existing_tiles: List[Tile], prefix: str, bucket: str = utils.get_bucket(), ignore_existing_tiles=False, ) -> List[Dict[str, Any]]: """Create geojson listing all tiles and upload to S3.""" response: List[Dict[str, Any]] = list() if ignore_existing_tiles: all_tiles = processed_tiles else: all_tiles = processed_tiles + existing_tiles # Upload a tiles.geojson for the default format even if no tiles dst_formats: Set[DstFormat] = {DstFormat(GLOBALS.default_dst_format)} for tile in all_tiles: for fmt in tile.dst.keys(): dst_formats.add(DstFormat(fmt)) for dst_format in dst_formats: fc: FeatureCollection = generate_feature_collection( all_tiles, dst_format) response.append( _upload_extent(fc, prefix=prefix, dst_format=dst_format)) key = os.path.join(prefix, dst_format, "tiles.geojson") response.append(_upload_geojson(fc, bucket, key)) return response
def upload(self) -> None: try: bucket = utils.get_bucket() for dst_format in self.local_dst.keys(): local_tiff_path = self.local_dst[dst_format].uri LOGGER.info(f"Upload {local_tiff_path} to s3") _ = upload_s3( local_tiff_path, bucket, self.dst[dst_format].uri, ) # Also upload the stats sidecar file that gdalinfo creates # Use the default format for path because we only create 1 sidecar local_stats_path = self.local_dst[ self.default_format].uri + stats_ext if os.path.isfile(local_stats_path): LOGGER.info(f"Upload {local_stats_path} to s3") _ = upload_s3( local_stats_path, bucket, self.dst[dst_format].uri + stats_ext, ) except Exception as e: LOGGER.error(f"Could not upload file {self.tile_id}") LOGGER.exception(str(e)) self.status = "failed"
def _upload_extent( fc: FeatureCollection, prefix: str, dst_format: str, bucket: str = utils.get_bucket(), ) -> Dict[str, Any]: """Create geojson file for tileset extent and upload to S3.""" extent_fc = _union_tile_geoms(fc) key = os.path.join(prefix, dst_format, "extent.geojson") return _upload_geojson(extent_fc, bucket, key)
def create_geojsons( resources: List[Tuple[str, str, str]], dataset: str, version: str, prefix: str, merge_existing: bool, ) -> None: get_files = {"s3": get_aws_files, "gs": get_gs_files} tiles: List[DummyTile] = list() for provider, bucket, key in resources: files = get_files[provider](bucket, key) for uri in files: src = RasterSource(uri) tiles.append(DummyTile({"geotiff": src})) data_lake_bucket = get_bucket() target_prefix = f"{dataset}/{version}/{prefix.strip('/')}/" # Don't bother checking for existing tiles unless we're going to use them existing_tiles = list() if merge_existing: existing_uris = get_aws_files(data_lake_bucket, target_prefix) for uri in existing_uris: src = RasterSource(uri) existing_tiles.append(DummyTile({"geotiff": src})) upload_geometries.upload_geojsons( tiles, # type: ignore existing_tiles, # type: ignore bucket=data_lake_bucket, prefix=target_prefix, ignore_existing_tiles=not merge_existing, )
def bucket(self): return get_bucket()