def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
body = requests.get(self.path).content
if gzip or (gzip is None and self.path.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
body = _aws_get_data(self.key, self.bucket, client=self.client)
if gzip or (gzip is None and self.key.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def test_compress():
"""Test valid gz compression."""
with open(mosaic_json, "r") as f:
mosaic = json.loads(f.read())
body = utils._compress_gz_json(mosaic)
assert type(body) == bytes
res = json.loads(utils._decompress_gz(body))
assert res == mosaic
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
with open(self.path, "rb") as f:
body = f.read()
if gzip or (gzip is None and self.path.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _read(self) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
body = self._get_object(self.key, self.bucket)
self._file_byte_size = len(body)
if self.key.endswith(".gz"):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def test_decompress():
"""Test valid gz decompression."""
with open(mosaic_gz, "rb") as f:
body = f.read()
res = json.loads(utils._decompress_gz(body))
assert list(res.keys()) == [
"mosaicjson",
"quadkey_zoom",
"minzoom",
"maxzoom",
"bounds",
"center",
"tiles",
]
def _read(self) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
try:
with open(self.path, "rb") as f:
body = f.read()
except Exception as e:
exc = _FILE_EXCEPTIONS.get(e, MosaicError) # type: ignore
raise exc(str(e)) from e
self._file_byte_size = len(body)
if self.path.endswith(".gz"):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
try:
r = requests.get(self.path)
r.raise_for_status()
except requests.exceptions.HTTPError as e:
# post-flight errors
status_code = e.response.status_code
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response.content) from e
except requests.exceptions.RequestException as e:
# pre-flight errors
raise MosaicError(e.args[0].reason) from e
body = r.content
self._file_byte_size = len(body)
if gzip or (gzip is None and self.path.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _read(self) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
try:
r = httpx.get(self.input)
r.raise_for_status()
except httpx.HTTPStatusError as e:
# post-flight errors
status_code = e.response.status_code
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response.content) from e
except httpx.RequestError as e:
# pre-flight errors
raise MosaicError(e.args[0].reason) from e
body = r.content
self._file_byte_size = len(body)
if self.input.endswith(".gz"):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def test_file_backend():
"""Test File backend."""
with MosaicBackend(mosaic_gz) as mosaic:
assert mosaic._backend_name == "File"
assert isinstance(mosaic, FileBackend)
assert (mosaic.mosaicid ==
"24d43802c19ef67cc498c327b62514ecf70c2bbb1bbc243dda1ee075")
assert mosaic.quadkey_zoom == 7
info = mosaic.info()
assert not info["quadkeys"]
assert list(info.dict()) == [
"bounds",
"center",
"minzoom",
"maxzoom",
"name",
"quadkeys",
]
info = mosaic.info(quadkeys=True)
assert info["quadkeys"]
assert list(mosaic.metadata.dict(exclude_none=True).keys()) == [
"mosaicjson",
"version",
"minzoom",
"maxzoom",
"quadkey_zoom",
"bounds",
"center",
]
assert mosaic.assets_for_tile(150, 182, 9) == ["cog1.tif", "cog2.tif"]
assert mosaic.assets_for_point(-73, 45) == ["cog1.tif", "cog2.tif"]
assert len(mosaic.get_assets(150, 182, 9)) == 2
assert len(mosaic.get_assets(147, 182, 12)) == 0
with MosaicBackend(mosaic_bin, backend_options={"gzip": True}) as mosaic:
assert isinstance(mosaic, FileBackend)
assert (mosaic.mosaicid ==
"24d43802c19ef67cc498c327b62514ecf70c2bbb1bbc243dda1ee075")
assert mosaic.quadkey_zoom == 7
with MosaicBackend(mosaic_json) as mosaic:
assert isinstance(mosaic, FileBackend)
assert mosaic.quadkey_zoom == 7
with MosaicBackend(mosaic_jsonV1) as mosaic:
assert isinstance(mosaic, FileBackend)
assert mosaic.quadkey_zoom == 7
assert list(mosaic.metadata.dict(exclude_none=True).keys()) == [
"mosaicjson",
"version",
"minzoom",
"maxzoom",
"bounds",
"center",
]
with pytest.raises(ValidationError):
with MosaicBackend("afile.json", mosaic_def={}):
pass
runner = CliRunner()
with runner.isolated_filesystem():
with MosaicBackend("mosaic.json", mosaic_def=mosaic_content) as mosaic:
mosaic.write()
with open("mosaic.json") as f:
m = json.loads(f.read())
assert m["quadkey_zoom"] == 7
with pytest.raises(MosaicExistsError):
mosaic.write()
mosaic.write(overwrite=True)
with MosaicBackend("mosaic.json.gz",
mosaic_def=mosaic_content) as mosaic:
mosaic.write()
with open("mosaic.json.gz", "rb") as f:
m = json.loads(_decompress_gz(f.read()))
assert m["quadkey_zoom"] == 7
with MosaicBackend("abinaryfile.bin",
mosaic_def=mosaic_content) as mosaic:
mosaic.write(gzip=True)
with open("abinaryfile.bin", "rb") as f:
m = json.loads(_decompress_gz(f.read()))
assert m["quadkey_zoom"] == 7
def test_s3_backend(session):
"""Test S3 backend."""
with open(mosaic_gz, "rb") as f:
session.return_value.client.return_value.get_object.return_value = {
"Body": BytesIO(f.read())
}
session.return_value.client.return_value.put_object.return_value = True
session.return_value.client.return_value.head_object.return_value = False
with MosaicBackend("s3://mybucket/mymosaic.json.gz") as mosaic:
assert mosaic._backend_name == "AWS S3"
assert isinstance(mosaic, S3Backend)
assert (mosaic.mosaicid ==
"24d43802c19ef67cc498c327b62514ecf70c2bbb1bbc243dda1ee075")
assert mosaic.quadkey_zoom == 7
assert list(mosaic.metadata.dict(exclude_none=True).keys()) == [
"mosaicjson",
"version",
"minzoom",
"maxzoom",
"quadkey_zoom",
"bounds",
"center",
]
assert mosaic.assets_for_tile(150, 182, 9) == ["cog1.tif", "cog2.tif"]
assert mosaic.assets_for_point(-73, 45) == ["cog1.tif", "cog2.tif"]
session.return_value.client.return_value.get_object.assert_called_once_with(
Bucket="mybucket", Key="mymosaic.json.gz")
session.return_value.client.return_value.put_object.assert_not_called()
session.return_value.client.return_value.head_object.assert_not_called(
)
session.reset_mock()
with open(mosaic_gz, "rb") as f:
session.return_value.client.return_value.get_object.return_value = {
"Body": BytesIO(f.read())
}
session.return_value.client.return_value.put_object.return_value = True
session.return_value.client.return_value.head_object.return_value = False
with MosaicBackend("s3://mybucket/mymosaic.json.gz",
mosaic_def=mosaic_content) as mosaic:
assert isinstance(mosaic, S3Backend)
mosaic.write()
session.return_value.client.return_value.get_object.assert_not_called()
session.return_value.client.return_value.head_object.assert_called_once()
kwargs = session.return_value.client.return_value.put_object.call_args[1]
assert kwargs["Bucket"] == "mybucket"
assert kwargs["Key"] == "mymosaic.json.gz"
assert MosaicJSON(**json.loads(_decompress_gz(kwargs["Body"])))
session.reset_mock()
session.return_value.client.return_value.head_object.return_value = False
with MosaicBackend("s3://mybucket/00000",
mosaic_def=mosaic_content) as mosaic:
assert isinstance(mosaic, S3Backend)
mosaic.write(gzip=True)
session.return_value.client.return_value.get_object.assert_not_called()
session.return_value.client.return_value.head_object.assert_called_once()
kwargs = session.return_value.client.return_value.put_object.call_args[1]
assert kwargs["Bucket"] == "mybucket"
assert kwargs["Key"] == "00000"
assert MosaicJSON(**json.loads(_decompress_gz(kwargs["Body"])))
session.reset_mock()
session.return_value.client.return_value.head_object.return_value = False
with MosaicBackend("s3://mybucket/00000",
mosaic_def=mosaic_content) as mosaic:
assert isinstance(mosaic, S3Backend)
mosaic.write()
session.return_value.client.return_value.get_object.assert_not_called()
session.return_value.client.return_value.head_object.assert_called_once()
kwargs = session.return_value.client.return_value.put_object.call_args[1]
assert kwargs["Bucket"] == "mybucket"
assert kwargs["Key"] == "00000"
assert MosaicJSON(**json.loads(kwargs["Body"]))
session.reset_mock()
session.return_value.client.return_value.head_object.return_value = True
with MosaicBackend("s3://mybucket/00000",
mosaic_def=mosaic_content) as mosaic:
assert isinstance(mosaic, S3Backend)
with pytest.raises(MosaicExistsError):
mosaic.write()
session.return_value.client.return_value.get_object.assert_not_called()
session.return_value.client.return_value.head_object.assert_called_once()
kwargs = session.return_value.client.return_value.put_object.assert_not_called(
)
session.reset_mock()
session.return_value.client.return_value.head_object.return_value = True
with MosaicBackend("s3://mybucket/00000",
mosaic_def=mosaic_content) as mosaic:
assert isinstance(mosaic, S3Backend)
mosaic.write(overwrite=True)
session.return_value.client.return_value.get_object.assert_not_called()
session.return_value.client.return_value.head_object.assert_not_called()
kwargs = session.return_value.client.return_value.put_object.assert_called_once(
)
session.reset_mock()
def test_file_backend():
"""Test File backend."""
with MosaicBackend(mosaic_gz) as mosaic:
assert isinstance(mosaic, FileBackend)
assert (mosaic.mosaicid ==
"f39f05644731addf1d183fa094ff6478900a27912ad035ef570231b1")
assert mosaic.quadkey_zoom == 7
assert list(mosaic.metadata.keys()) == [
"mosaicjson",
"version",
"minzoom",
"maxzoom",
"quadkey_zoom",
"bounds",
"center",
]
assert mosaic.tile(150, 182, 9) == ["cog1.tif", "cog2.tif"]
assert mosaic.point(-73, 45) == ["cog1.tif", "cog2.tif"]
with MosaicBackend(mosaic_bin, gzip=True) as mosaic:
assert isinstance(mosaic, FileBackend)
assert (mosaic.mosaicid ==
"f39f05644731addf1d183fa094ff6478900a27912ad035ef570231b1")
assert mosaic.quadkey_zoom == 7
with MosaicBackend(mosaic_json) as mosaic:
assert isinstance(mosaic, FileBackend)
assert mosaic.quadkey_zoom == 7
with MosaicBackend(mosaic_jsonV1) as mosaic:
assert isinstance(mosaic, FileBackend)
assert mosaic.quadkey_zoom == 7
assert list(mosaic.metadata.keys()) == [
"mosaicjson",
"version",
"minzoom",
"maxzoom",
"bounds",
"center",
]
runner = CliRunner()
with runner.isolated_filesystem():
with MosaicBackend("mosaic.json", mosaic_def=mosaic_content) as mosaic:
mosaic.write()
with open("mosaic.json") as f:
m = json.loads(f.read())
print
assert m["quadkey_zoom"] == 7
with MosaicBackend("mosaic.json.gz",
mosaic_def=mosaic_content) as mosaic:
mosaic.write()
with open("mosaic.json.gz", "rb") as f:
m = json.loads(_decompress_gz(f.read()))
assert m["quadkey_zoom"] == 7
with MosaicBackend("abinaryfile.bin",
mosaic_def=mosaic_content) as mosaic:
mosaic.write(gzip=True)
with open("abinaryfile.bin", "rb") as f:
m = json.loads(_decompress_gz(f.read()))
assert m["quadkey_zoom"] == 7
def test_s3_backend(session):
"""Test S3 backend."""
with open(mosaic_gz, "rb") as f:
session.return_value.client.return_value.get_object.return_value = {
"Body": BytesIO(f.read())
}
session.return_value.client.return_value.put_object.return_value = True
with MosaicBackend("s3://mybucket/mymosaic.json.gz") as mosaic:
assert isinstance(mosaic, S3Backend)
assert (mosaic.mosaicid ==
"f39f05644731addf1d183fa094ff6478900a27912ad035ef570231b1")
assert mosaic.quadkey_zoom == 7
assert list(mosaic.metadata.keys()) == [
"mosaicjson",
"version",
"minzoom",
"maxzoom",
"quadkey_zoom",
"bounds",
"center",
]
assert mosaic.tile(150, 182, 9) == ["cog1.tif", "cog2.tif"]
assert mosaic.point(-73, 45) == ["cog1.tif", "cog2.tif"]
session.return_value.client.return_value.get_object.assert_called_once_with(
Bucket="mybucket", Key="mymosaic.json.gz")
session.return_value.client.return_value.put_object.assert_not_called()
session.reset_mock()
with open(mosaic_gz, "rb") as f:
session.return_value.client.return_value.get_object.return_value = {
"Body": BytesIO(f.read())
}
session.return_value.client.return_value.put_object.return_value = True
with MosaicBackend("s3://mybucket/mymosaic.json.gz",
mosaic_def=mosaic_content) as mosaic:
assert isinstance(mosaic, S3Backend)
mosaic.write()
session.return_value.client.return_value.get_object.assert_not_called()
kwargs = session.return_value.client.return_value.put_object.call_args[1]
assert kwargs["Bucket"] == "mybucket"
assert kwargs["Key"] == "mymosaic.json.gz"
assert MosaicJSON(**json.loads(_decompress_gz(kwargs["Body"])))
session.reset_mock()
with MosaicBackend("s3://mybucket/00000",
mosaic_def=mosaic_content) as mosaic:
assert isinstance(mosaic, S3Backend)
mosaic.write(gzip=True)
session.return_value.client.return_value.get_object.assert_not_called()
kwargs = session.return_value.client.return_value.put_object.call_args[1]
assert kwargs["Bucket"] == "mybucket"
assert kwargs["Key"] == "00000"
assert MosaicJSON(**json.loads(_decompress_gz(kwargs["Body"])))
session.reset_mock()
with MosaicBackend("s3://mybucket/00000",
mosaic_def=mosaic_content) as mosaic:
assert isinstance(mosaic, S3Backend)
mosaic.write()
session.return_value.client.return_value.get_object.assert_not_called()
kwargs = session.return_value.client.return_value.put_object.call_args[1]
assert kwargs["Bucket"] == "mybucket"
assert kwargs["Key"] == "00000"
assert MosaicJSON(**json.loads(kwargs["Body"]))
session.reset_mock()
def test_file_backend():
"""Test File backend."""
with MosaicBackend(mosaic_gz) as mosaic:
assert mosaic._backend_name == "File"
assert isinstance(mosaic, FileBackend)
assert (mosaic.mosaicid ==
"24d43802c19ef67cc498c327b62514ecf70c2bbb1bbc243dda1ee075")
assert mosaic.quadkey_zoom == 7
assert mosaic.minzoom == mosaic.mosaic_def.minzoom
info = mosaic.info()
assert not info["quadkeys"]
assert list(info.dict()) == [
"bounds",
"center",
"minzoom",
"maxzoom",
"name",
"quadkeys",
]
info = mosaic.info(quadkeys=True)
assert info["quadkeys"]
assert list(
mosaic.mosaic_def.dict(exclude_none=True,
exclude={"tiles"}).keys()) == [
"mosaicjson",
"version",
"minzoom",
"maxzoom",
"quadkey_zoom",
"bounds",
"center",
]
# make sure we do not return asset twice (e.g for parent tile)
assert mosaic.assets_for_tile(18, 22, 6) == ["cog1.tif", "cog2.tif"]
assert mosaic.assets_for_tile(150, 182, 9) == ["cog1.tif", "cog2.tif"]
assert mosaic.assets_for_point(-73, 45) == ["cog1.tif", "cog2.tif"]
assert len(mosaic.get_assets(150, 182, 9)) == 2
assert len(mosaic.get_assets(147, 182, 12)) == 0
assert mosaic.assets_for_bbox(
-74.53125, 45.583289756006316, -73.828125,
46.07323062540836) == ["cog1.tif", "cog2.tif"]
with MosaicBackend(mosaic_json) as mosaic:
assert isinstance(mosaic, FileBackend)
assert mosaic.quadkey_zoom == 7
with MosaicBackend(mosaic_jsonV1) as mosaic:
assert isinstance(mosaic, FileBackend)
assert mosaic.quadkey_zoom == 7
assert list(
mosaic.mosaic_def.dict(exclude_none=True,
exclude={"tiles"}).keys()) == [
"mosaicjson", "version", "minzoom",
"maxzoom", "bounds", "center"
]
with pytest.raises(ValidationError):
with MosaicBackend("afile.json", mosaic_def={}):
pass
runner = CliRunner()
with runner.isolated_filesystem():
with MosaicBackend("mosaic.json", mosaic_def=mosaic_content) as mosaic:
mosaic.write()
assert mosaic.minzoom == mosaic_content["minzoom"]
with open("mosaic.json") as f:
m = json.loads(f.read())
assert m["quadkey_zoom"] == 7
with pytest.raises(MosaicExistsError):
mosaic.write()
mosaic.write(overwrite=True)
with MosaicBackend("mosaic.json.gz",
mosaic_def=mosaic_content) as mosaic:
mosaic.write()
with open("mosaic.json.gz", "rb") as f:
m = json.loads(_decompress_gz(f.read()))
assert m["quadkey_zoom"] == 7
mosaic_oneasset = MosaicJSON.from_urls([asset1], quiet=True)
with MosaicBackend("umosaic.json.gz",
mosaic_def=mosaic_oneasset) as mosaic:
mosaic.write()
assert len(mosaic.get_assets(150, 182, 9)) == 1
with MosaicBackend("umosaic.json.gz") as mosaic:
features = get_footprints([asset2], quiet=True)
mosaic.update(features)
assets = mosaic.get_assets(150, 182, 9)
assert len(assets) == 2
assert assets[0] == asset2
assert assets[1] == asset1
