def write(self, process_tile, data): """ Write data from process tiles into PNG file(s). Parameters ---------- process_tile : ``BufferedTile`` must be member of process ``TilePyramid`` """ data = self._prepare_array(data) if data.mask.all(): # pragma: no cover logger.debug("data empty, nothing to write") else: # in case of S3 output, create an boto3 resource bucket_resource = get_boto3_bucket( self._bucket) if self._bucket else None # Convert from process_tile to output_tiles and write for tile in self.pyramid.intersecting(process_tile): out_path = self.get_path(tile) self.prepare_path(tile) out_tile = BufferedTile(tile, self.pixelbuffer) write_raster_window(in_tile=process_tile, in_data=data, out_profile=self.profile(out_tile), out_tile=out_tile, out_path=out_path, bucket_resource=bucket_resource)
def gen_indexes_and_check(): # generate indexes list(zoom_index_gen( mp=mp, zoom=zoom, out_dir=mp.config.output.path, geojson=True, txt=True, vrt=True )) # assert GeoJSON exists with fiona.open(os.path.join(mp.config.output.path, "%s.geojson" % zoom)) as src: assert len(src) == 2 # assert TXT exists txt_index = os.path.join(mp.config.output.path, "%s.txt" % zoom) bucket = get_boto3_bucket(txt_index.split("/")[2]) key = "/".join(txt_index.split("/")[3:]) for obj in bucket.objects.filter(Prefix=key): if obj.key == key: content = obj.get()['Body'].read().decode() assert len([l + '\n' for l in content.split('\n') if l]) == 2 # assert VRT exists with rasterio.open(os.path.join(mp.config.output.path, "%s.vrt" % zoom)) as src: assert src.read().any()
def write(self, process_tile, data): """ Write data from process tiles into GeoJSON file(s). Parameters ---------- process_tile : ``BufferedTile`` must be member of process ``TilePyramid`` """ if data is None or len(data) == 0: return if not isinstance(data, (list, types.GeneratorType)): raise TypeError( "GeoJSON driver data has to be a list or generator of GeoJSON objects" ) data = list(data) if not len(data): logger.debug("no features to write") else: # in case of S3 output, create an boto3 resource bucket_resource = get_boto3_bucket( self._bucket) if self._bucket else None # Convert from process_tile to output_tiles for tile in self.pyramid.intersecting(process_tile): out_path = self.get_path(tile) self.prepare_path(tile) out_tile = BufferedTile(tile, self.pixelbuffer) write_vector_window(in_data=data, out_schema=self.output_params["schema"], out_tile=out_tile, out_path=out_path, bucket_resource=bucket_resource)
def __init__(self, out_path=None, output=None, out_pyramid=None): # see if lxml is installed before checking all output tiles from lxml.builder import ElementMaker self.path = out_path self._tp = out_pyramid self._output = output self._bucket = self.path.split("/")[2] if self.path.startswith( "s3://") else None self.bucket_resource = get_boto3_bucket( self._bucket) if self._bucket else None logger.debug("initialize VRT writer for %s", self.path) if path_exists(self.path): if self._bucket: key = "/".join(self.path.split("/")[3:]) for obj in self.bucket_resource.objects.filter(Prefix=key): if obj.key == key: self._existing = { k: v for k, v in self._xml_to_entries( obj.get()['Body'].read().decode()) } else: with open(self.path) as src: self._existing = { k: v for k, v in self._xml_to_entries(src.read()) } else: self._existing = {} logger.debug("%s existing entries", len(self._existing)) self.new_entries = 0 self._new = {}
def __init__(self, out_path=None): self.path = out_path self._bucket = self.path.split("/")[2] if self.path.startswith( "s3://") else None self.bucket_resource = get_boto3_bucket( self._bucket) if self._bucket else None logger.debug("initialize TXT writer") if path_exists(self.path): if self._bucket: key = "/".join(self.path.split("/")[3:]) for obj in self.bucket_resource.objects.filter(Prefix=key): if obj.key == key: self._existing = { l + '\n' for l in obj.get()['Body'].read().decode().split( '\n') if l } else: with open(self.path) as src: self._existing = {l for l in src} else: self._existing = {} self.new_entries = 0 if self._bucket: self.sink = "" else: self.sink = open(self.path, "w") for l in self._existing: self._write_line(l)
def write(self, process_tile, data): """ Write data from process tiles into GeoTIFF file(s). Parameters ---------- process_tile : ``BufferedTile`` must be member of process ``TilePyramid`` data : ``np.ndarray`` """ if ( isinstance(data, tuple) and len(data) == 2 and isinstance(data[1], dict) ): data, tags = data else: tags = {} data = prepare_array( data, masked=True, nodata=self.output_params["nodata"], dtype=self.profile(process_tile)["dtype"] ) if data.mask.all(): logger.debug("data empty, nothing to write") else: # in case of S3 output, create an boto3 resource bucket_resource = get_boto3_bucket(self._bucket) if self._bucket else None # Convert from process_tile to output_tiles and write for tile in self.pyramid.intersecting(process_tile): out_path = self.get_path(tile) self.prepare_path(tile) out_tile = BufferedTile(tile, self.pixelbuffer) write_raster_window( in_tile=process_tile, in_data=data, out_profile=self.profile(out_tile), out_tile=out_tile, out_path=out_path, tags=tags, bucket_resource=bucket_resource )
def __init__(self, output_params, **kwargs): """Initialize.""" logger.debug("output is single file") self.dst = None super().__init__(output_params, **kwargs) self._set_attributes(output_params) if len(self.output_params["delimiters"]["zoom"]) != 1: raise ValueError( "single file output only works with one zoom level") self.zoom = output_params["delimiters"]["zoom"][0] self.cog = output_params.get("cog", False) if self.cog or "overviews" in output_params: self.overviews = True self.overviews_resampling = output_params.get( "overviews_resampling", "nearest") self.overviews_levels = output_params.get( "overviews_levels", [2**i for i in range(1, self.zoom + 1)]) else: self.overviews = False self.in_memory = output_params.get("in_memory", True) _bucket = self.path.split("/")[2] if self.path.startswith( "s3://") else None self._bucket_resource = get_boto3_bucket(_bucket) if _bucket else None