def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" body = requests.get(self.path).content if gzip or (gzip is None and self.path.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" body = _aws_get_data(self.key, self.bucket, client=self.client) if gzip or (gzip is None and self.key.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def test_compress(): """Test valid gz compression.""" with open(mosaic_json, "r") as f: mosaic = json.loads(f.read()) body = utils._compress_gz_json(mosaic) assert type(body) == bytes res = json.loads(utils._decompress_gz(body)) assert res == mosaic
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" with open(self.path, "rb") as f: body = f.read() if gzip or (gzip is None and self.path.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def _read(self) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" body = self._get_object(self.key, self.bucket) self._file_byte_size = len(body) if self.key.endswith(".gz"): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def test_decompress(): """Test valid gz decompression.""" with open(mosaic_gz, "rb") as f: body = f.read() res = json.loads(utils._decompress_gz(body)) assert list(res.keys()) == [ "mosaicjson", "quadkey_zoom", "minzoom", "maxzoom", "bounds", "center", "tiles", ]
def _read(self) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" try: with open(self.path, "rb") as f: body = f.read() except Exception as e: exc = _FILE_EXCEPTIONS.get(e, MosaicError) # type: ignore raise exc(str(e)) from e self._file_byte_size = len(body) if self.path.endswith(".gz"): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" try: r = requests.get(self.path) r.raise_for_status() except requests.exceptions.HTTPError as e: # post-flight errors status_code = e.response.status_code exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) raise exc(e.response.content) from e except requests.exceptions.RequestException as e: # pre-flight errors raise MosaicError(e.args[0].reason) from e body = r.content self._file_byte_size = len(body) if gzip or (gzip is None and self.path.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def _read(self) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" try: r = httpx.get(self.input) r.raise_for_status() except httpx.HTTPStatusError as e: # post-flight errors status_code = e.response.status_code exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) raise exc(e.response.content) from e except httpx.RequestError as e: # pre-flight errors raise MosaicError(e.args[0].reason) from e body = r.content self._file_byte_size = len(body) if self.input.endswith(".gz"): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def test_file_backend(): """Test File backend.""" with MosaicBackend(mosaic_gz) as mosaic: assert mosaic._backend_name == "File" assert isinstance(mosaic, FileBackend) assert (mosaic.mosaicid == "24d43802c19ef67cc498c327b62514ecf70c2bbb1bbc243dda1ee075") assert mosaic.quadkey_zoom == 7 info = mosaic.info() assert not info["quadkeys"] assert list(info.dict()) == [ "bounds", "center", "minzoom", "maxzoom", "name", "quadkeys", ] info = mosaic.info(quadkeys=True) assert info["quadkeys"] assert list(mosaic.metadata.dict(exclude_none=True).keys()) == [ "mosaicjson", "version", "minzoom", "maxzoom", "quadkey_zoom", "bounds", "center", ] assert mosaic.assets_for_tile(150, 182, 9) == ["cog1.tif", "cog2.tif"] assert mosaic.assets_for_point(-73, 45) == ["cog1.tif", "cog2.tif"] assert len(mosaic.get_assets(150, 182, 9)) == 2 assert len(mosaic.get_assets(147, 182, 12)) == 0 with MosaicBackend(mosaic_bin, backend_options={"gzip": True}) as mosaic: assert isinstance(mosaic, FileBackend) assert (mosaic.mosaicid == "24d43802c19ef67cc498c327b62514ecf70c2bbb1bbc243dda1ee075") assert mosaic.quadkey_zoom == 7 with MosaicBackend(mosaic_json) as mosaic: assert isinstance(mosaic, FileBackend) assert mosaic.quadkey_zoom == 7 with MosaicBackend(mosaic_jsonV1) as mosaic: assert isinstance(mosaic, FileBackend) assert mosaic.quadkey_zoom == 7 assert list(mosaic.metadata.dict(exclude_none=True).keys()) == [ "mosaicjson", "version", "minzoom", "maxzoom", "bounds", "center", ] with pytest.raises(ValidationError): with MosaicBackend("afile.json", mosaic_def={}): pass runner = CliRunner() with runner.isolated_filesystem(): with MosaicBackend("mosaic.json", mosaic_def=mosaic_content) as mosaic: mosaic.write() with open("mosaic.json") as f: m = json.loads(f.read()) assert m["quadkey_zoom"] == 7 with pytest.raises(MosaicExistsError): mosaic.write() mosaic.write(overwrite=True) with MosaicBackend("mosaic.json.gz", mosaic_def=mosaic_content) as mosaic: mosaic.write() with open("mosaic.json.gz", "rb") as f: m = json.loads(_decompress_gz(f.read())) assert m["quadkey_zoom"] == 7 with MosaicBackend("abinaryfile.bin", mosaic_def=mosaic_content) as mosaic: mosaic.write(gzip=True) with open("abinaryfile.bin", "rb") as f: m = json.loads(_decompress_gz(f.read())) assert m["quadkey_zoom"] == 7
def test_s3_backend(session): """Test S3 backend.""" with open(mosaic_gz, "rb") as f: session.return_value.client.return_value.get_object.return_value = { "Body": BytesIO(f.read()) } session.return_value.client.return_value.put_object.return_value = True session.return_value.client.return_value.head_object.return_value = False with MosaicBackend("s3://mybucket/mymosaic.json.gz") as mosaic: assert mosaic._backend_name == "AWS S3" assert isinstance(mosaic, S3Backend) assert (mosaic.mosaicid == "24d43802c19ef67cc498c327b62514ecf70c2bbb1bbc243dda1ee075") assert mosaic.quadkey_zoom == 7 assert list(mosaic.metadata.dict(exclude_none=True).keys()) == [ "mosaicjson", "version", "minzoom", "maxzoom", "quadkey_zoom", "bounds", "center", ] assert mosaic.assets_for_tile(150, 182, 9) == ["cog1.tif", "cog2.tif"] assert mosaic.assets_for_point(-73, 45) == ["cog1.tif", "cog2.tif"] session.return_value.client.return_value.get_object.assert_called_once_with( Bucket="mybucket", Key="mymosaic.json.gz") session.return_value.client.return_value.put_object.assert_not_called() session.return_value.client.return_value.head_object.assert_not_called( ) session.reset_mock() with open(mosaic_gz, "rb") as f: session.return_value.client.return_value.get_object.return_value = { "Body": BytesIO(f.read()) } session.return_value.client.return_value.put_object.return_value = True session.return_value.client.return_value.head_object.return_value = False with MosaicBackend("s3://mybucket/mymosaic.json.gz", mosaic_def=mosaic_content) as mosaic: assert isinstance(mosaic, S3Backend) mosaic.write() session.return_value.client.return_value.get_object.assert_not_called() session.return_value.client.return_value.head_object.assert_called_once() kwargs = session.return_value.client.return_value.put_object.call_args[1] assert kwargs["Bucket"] == "mybucket" assert kwargs["Key"] == "mymosaic.json.gz" assert MosaicJSON(**json.loads(_decompress_gz(kwargs["Body"]))) session.reset_mock() session.return_value.client.return_value.head_object.return_value = False with MosaicBackend("s3://mybucket/00000", mosaic_def=mosaic_content) as mosaic: assert isinstance(mosaic, S3Backend) mosaic.write(gzip=True) session.return_value.client.return_value.get_object.assert_not_called() session.return_value.client.return_value.head_object.assert_called_once() kwargs = session.return_value.client.return_value.put_object.call_args[1] assert kwargs["Bucket"] == "mybucket" assert kwargs["Key"] == "00000" assert MosaicJSON(**json.loads(_decompress_gz(kwargs["Body"]))) session.reset_mock() session.return_value.client.return_value.head_object.return_value = False with MosaicBackend("s3://mybucket/00000", mosaic_def=mosaic_content) as mosaic: assert isinstance(mosaic, S3Backend) mosaic.write() session.return_value.client.return_value.get_object.assert_not_called() session.return_value.client.return_value.head_object.assert_called_once() kwargs = session.return_value.client.return_value.put_object.call_args[1] assert kwargs["Bucket"] == "mybucket" assert kwargs["Key"] == "00000" assert MosaicJSON(**json.loads(kwargs["Body"])) session.reset_mock() session.return_value.client.return_value.head_object.return_value = True with MosaicBackend("s3://mybucket/00000", mosaic_def=mosaic_content) as mosaic: assert isinstance(mosaic, S3Backend) with pytest.raises(MosaicExistsError): mosaic.write() session.return_value.client.return_value.get_object.assert_not_called() session.return_value.client.return_value.head_object.assert_called_once() kwargs = session.return_value.client.return_value.put_object.assert_not_called( ) session.reset_mock() session.return_value.client.return_value.head_object.return_value = True with MosaicBackend("s3://mybucket/00000", mosaic_def=mosaic_content) as mosaic: assert isinstance(mosaic, S3Backend) mosaic.write(overwrite=True) session.return_value.client.return_value.get_object.assert_not_called() session.return_value.client.return_value.head_object.assert_not_called() kwargs = session.return_value.client.return_value.put_object.assert_called_once( ) session.reset_mock()
def test_file_backend(): """Test File backend.""" with MosaicBackend(mosaic_gz) as mosaic: assert isinstance(mosaic, FileBackend) assert (mosaic.mosaicid == "f39f05644731addf1d183fa094ff6478900a27912ad035ef570231b1") assert mosaic.quadkey_zoom == 7 assert list(mosaic.metadata.keys()) == [ "mosaicjson", "version", "minzoom", "maxzoom", "quadkey_zoom", "bounds", "center", ] assert mosaic.tile(150, 182, 9) == ["cog1.tif", "cog2.tif"] assert mosaic.point(-73, 45) == ["cog1.tif", "cog2.tif"] with MosaicBackend(mosaic_bin, gzip=True) as mosaic: assert isinstance(mosaic, FileBackend) assert (mosaic.mosaicid == "f39f05644731addf1d183fa094ff6478900a27912ad035ef570231b1") assert mosaic.quadkey_zoom == 7 with MosaicBackend(mosaic_json) as mosaic: assert isinstance(mosaic, FileBackend) assert mosaic.quadkey_zoom == 7 with MosaicBackend(mosaic_jsonV1) as mosaic: assert isinstance(mosaic, FileBackend) assert mosaic.quadkey_zoom == 7 assert list(mosaic.metadata.keys()) == [ "mosaicjson", "version", "minzoom", "maxzoom", "bounds", "center", ] runner = CliRunner() with runner.isolated_filesystem(): with MosaicBackend("mosaic.json", mosaic_def=mosaic_content) as mosaic: mosaic.write() with open("mosaic.json") as f: m = json.loads(f.read()) print assert m["quadkey_zoom"] == 7 with MosaicBackend("mosaic.json.gz", mosaic_def=mosaic_content) as mosaic: mosaic.write() with open("mosaic.json.gz", "rb") as f: m = json.loads(_decompress_gz(f.read())) assert m["quadkey_zoom"] == 7 with MosaicBackend("abinaryfile.bin", mosaic_def=mosaic_content) as mosaic: mosaic.write(gzip=True) with open("abinaryfile.bin", "rb") as f: m = json.loads(_decompress_gz(f.read())) assert m["quadkey_zoom"] == 7
def test_s3_backend(session): """Test S3 backend.""" with open(mosaic_gz, "rb") as f: session.return_value.client.return_value.get_object.return_value = { "Body": BytesIO(f.read()) } session.return_value.client.return_value.put_object.return_value = True with MosaicBackend("s3://mybucket/mymosaic.json.gz") as mosaic: assert isinstance(mosaic, S3Backend) assert (mosaic.mosaicid == "f39f05644731addf1d183fa094ff6478900a27912ad035ef570231b1") assert mosaic.quadkey_zoom == 7 assert list(mosaic.metadata.keys()) == [ "mosaicjson", "version", "minzoom", "maxzoom", "quadkey_zoom", "bounds", "center", ] assert mosaic.tile(150, 182, 9) == ["cog1.tif", "cog2.tif"] assert mosaic.point(-73, 45) == ["cog1.tif", "cog2.tif"] session.return_value.client.return_value.get_object.assert_called_once_with( Bucket="mybucket", Key="mymosaic.json.gz") session.return_value.client.return_value.put_object.assert_not_called() session.reset_mock() with open(mosaic_gz, "rb") as f: session.return_value.client.return_value.get_object.return_value = { "Body": BytesIO(f.read()) } session.return_value.client.return_value.put_object.return_value = True with MosaicBackend("s3://mybucket/mymosaic.json.gz", mosaic_def=mosaic_content) as mosaic: assert isinstance(mosaic, S3Backend) mosaic.write() session.return_value.client.return_value.get_object.assert_not_called() kwargs = session.return_value.client.return_value.put_object.call_args[1] assert kwargs["Bucket"] == "mybucket" assert kwargs["Key"] == "mymosaic.json.gz" assert MosaicJSON(**json.loads(_decompress_gz(kwargs["Body"]))) session.reset_mock() with MosaicBackend("s3://mybucket/00000", mosaic_def=mosaic_content) as mosaic: assert isinstance(mosaic, S3Backend) mosaic.write(gzip=True) session.return_value.client.return_value.get_object.assert_not_called() kwargs = session.return_value.client.return_value.put_object.call_args[1] assert kwargs["Bucket"] == "mybucket" assert kwargs["Key"] == "00000" assert MosaicJSON(**json.loads(_decompress_gz(kwargs["Body"]))) session.reset_mock() with MosaicBackend("s3://mybucket/00000", mosaic_def=mosaic_content) as mosaic: assert isinstance(mosaic, S3Backend) mosaic.write() session.return_value.client.return_value.get_object.assert_not_called() kwargs = session.return_value.client.return_value.put_object.call_args[1] assert kwargs["Bucket"] == "mybucket" assert kwargs["Key"] == "00000" assert MosaicJSON(**json.loads(kwargs["Body"])) session.reset_mock()
def test_file_backend(): """Test File backend.""" with MosaicBackend(mosaic_gz) as mosaic: assert mosaic._backend_name == "File" assert isinstance(mosaic, FileBackend) assert (mosaic.mosaicid == "24d43802c19ef67cc498c327b62514ecf70c2bbb1bbc243dda1ee075") assert mosaic.quadkey_zoom == 7 assert mosaic.minzoom == mosaic.mosaic_def.minzoom info = mosaic.info() assert not info["quadkeys"] assert list(info.dict()) == [ "bounds", "center", "minzoom", "maxzoom", "name", "quadkeys", ] info = mosaic.info(quadkeys=True) assert info["quadkeys"] assert list( mosaic.mosaic_def.dict(exclude_none=True, exclude={"tiles"}).keys()) == [ "mosaicjson", "version", "minzoom", "maxzoom", "quadkey_zoom", "bounds", "center", ] # make sure we do not return asset twice (e.g for parent tile) assert mosaic.assets_for_tile(18, 22, 6) == ["cog1.tif", "cog2.tif"] assert mosaic.assets_for_tile(150, 182, 9) == ["cog1.tif", "cog2.tif"] assert mosaic.assets_for_point(-73, 45) == ["cog1.tif", "cog2.tif"] assert len(mosaic.get_assets(150, 182, 9)) == 2 assert len(mosaic.get_assets(147, 182, 12)) == 0 assert mosaic.assets_for_bbox( -74.53125, 45.583289756006316, -73.828125, 46.07323062540836) == ["cog1.tif", "cog2.tif"] with MosaicBackend(mosaic_json) as mosaic: assert isinstance(mosaic, FileBackend) assert mosaic.quadkey_zoom == 7 with MosaicBackend(mosaic_jsonV1) as mosaic: assert isinstance(mosaic, FileBackend) assert mosaic.quadkey_zoom == 7 assert list( mosaic.mosaic_def.dict(exclude_none=True, exclude={"tiles"}).keys()) == [ "mosaicjson", "version", "minzoom", "maxzoom", "bounds", "center" ] with pytest.raises(ValidationError): with MosaicBackend("afile.json", mosaic_def={}): pass runner = CliRunner() with runner.isolated_filesystem(): with MosaicBackend("mosaic.json", mosaic_def=mosaic_content) as mosaic: mosaic.write() assert mosaic.minzoom == mosaic_content["minzoom"] with open("mosaic.json") as f: m = json.loads(f.read()) assert m["quadkey_zoom"] == 7 with pytest.raises(MosaicExistsError): mosaic.write() mosaic.write(overwrite=True) with MosaicBackend("mosaic.json.gz", mosaic_def=mosaic_content) as mosaic: mosaic.write() with open("mosaic.json.gz", "rb") as f: m = json.loads(_decompress_gz(f.read())) assert m["quadkey_zoom"] == 7 mosaic_oneasset = MosaicJSON.from_urls([asset1], quiet=True) with MosaicBackend("umosaic.json.gz", mosaic_def=mosaic_oneasset) as mosaic: mosaic.write() assert len(mosaic.get_assets(150, 182, 9)) == 1 with MosaicBackend("umosaic.json.gz") as mosaic: features = get_footprints([asset2], quiet=True) mosaic.update(features) assets = mosaic.get_assets(150, 182, 9) assert len(assets) == 2 assert assets[0] == asset2 assert assets[1] == asset1