def test_relative_url(self):
_, name, baseurl = resolve_url("def", "https://github.com/abc")
self.assertEqual(name, "def")
self.assertEqual(baseurl, "https://github.com/abc")
# even with a path separator in the relative path, it should work.
_, name, baseurl = resolve_url("abc/def", "https://github.com/")
self.assertEqual(name, "def")
self.assertEqual(baseurl, "https://github.com/abc")
def test_fully_qualified_url(self):
_, name, baseurl = resolve_url("https://github.com/abc/def")
self.assertEqual(name, "def")
self.assertEqual(baseurl, "https://github.com/abc")
# even with a baseurl, this should work.
_, name, baseurl = resolve_url("https://github.com/abc/def",
"https://github.io")
self.assertEqual(name, "def")
self.assertEqual(baseurl, "https://github.com/abc")
def parse_doc(name_or_url, baseurl, backend_config=None):
backend, name, baseurl = resolve_url(name_or_url, baseurl,
backend_config)
with backend.read_contextmanager(name) as fh:
reader = codecs.getreader("utf-8")
json_doc = json.load(reader(fh))
try:
doc_version = version.parse(json_doc[CommonPartitionKeys.VERSION])
except KeyError as ex:
raise KeyError("JSON document missing `version` field. "
"Please specify the file format version.") from ex
try:
for version_cls in VERSIONS:
if version_cls.Reader.can_parse(doc_version):
parser = version_cls.Reader()
break
else:
raise ValueError("Unrecognized version number")
except KeyError:
raise KeyError(
"JSON document missing `version` field. Please specify the file format version."
)
return parser.parse(json_doc, baseurl, backend_config)
def write_to_url(partition: Union[Collection, TileSet],
url: str,
pretty: bool = False,
version_class=None,
*args,
**kwargs):
if version_class is None:
version_class = VERSIONS[-1]
document = version_class.Writer().generate_partition_document(
partition, url, pretty, *args, **kwargs)
indent = 4 if pretty else None
backend, name, _ = resolve_url(url)
with backend.write_file_handle(name) as fh:
writer = cast(TextIO, codecs.getwriter("utf-8")(fh))
json.dump(document,
writer,
indent=indent,
sort_keys=pretty,
ensure_ascii=False)
def write_tile(
self,
tile_url: str,
tile: Tile,
tile_format: ImageFormat,
backend_config: Optional[Mapping] = None,
) -> str:
"""Write the data for a tile to a given URL.
Parameters
----------
tile_url : str
The URL of the tile.
tile : Tile
The tile to be written.
tile_format : ImageFormat
The format to write the tile in.
backend_config : Optional[Mapping]
Mapping from the backend names to the config
Returns
-------
str :
The sha256 of the tile being added.
"""
backend, name, _ = resolve_url(tile_url, backend_config=backend_config)
buffer_fh = BytesIO()
tile.write(buffer_fh, tile_format)
buffer_fh.seek(0)
sha256 = hashlib.sha256(buffer_fh.getvalue()).hexdigest()
buffer_fh.seek(0)
with backend.write_file_handle(name) as fh:
fh.write(buffer_fh.read())
return sha256
def parse(self, json_doc, baseurl, backend_config):
if CollectionKeys.CONTENTS in json_doc:
# this is a Collection
result = Collection(
json_doc.get(CommonPartitionKeys.EXTRAS, None))
tp = ThreadPool()
try:
func = _base._parse_collection(_base.Reader.parse_doc,
baseurl, backend_config)
results = tp.map(func,
json_doc[CollectionKeys.CONTENTS].items())
finally:
tp.terminate()
for name, partition in results:
result.add_partition(name, partition)
elif TileSetKeys.TILES in json_doc:
imageformat = json_doc.get(TileSetKeys.DEFAULT_TILE_FORMAT,
None)
if imageformat is not None:
imageformat = ImageFormat[imageformat]
result = TileSet(
tuple(json_doc[TileSetKeys.DIMENSIONS]),
json_doc[TileSetKeys.SHAPE],
Tile.format_tuple_shape_to_dict_shape(
json_doc.get(TileSetKeys.DEFAULT_TILE_SHAPE, None)),
imageformat,
json_doc.get(TileSetKeys.EXTRAS, None),
)
for tile_doc in json_doc[TileSetKeys.TILES]:
relative_path_or_url = tile_doc[TileKeys.FILE]
backend, name, _ = resolve_url(relative_path_or_url,
baseurl, backend_config)
tile_format_str = tile_doc.get(TileKeys.TILE_FORMAT, None)
if tile_format_str:
tile_format = ImageFormat[tile_format_str]
else:
tile_format = result.default_tile_format
if tile_format is None:
# Still none :(
extension = os.path.splitext(name)[1].lstrip(".")
tile_format = ImageFormat.find_by_extension(extension)
checksum = tile_doc.get(TileKeys.SHA256, None)
tile = Tile(
tile_doc[TileKeys.COORDINATES],
tile_doc[TileKeys.INDICES],
tile_shape=Tile.format_tuple_shape_to_dict_shape(
tile_doc.get(TileKeys.TILE_SHAPE, None)),
sha256=checksum,
extras=tile_doc.get(TileKeys.EXTRAS, None),
)
tile.set_numpy_array_future(
SourceFileFuture(
backend.read_contextmanager(
name, checksum_sha256=checksum), tile_format))
result.add_tile(tile)
else:
raise ValueError(
"JSON doc does not appear to be a collection partition or a tileset "
"partition. JSON doc must contain either a {contents} field pointing to a "
"tile manifest, or it must contain a {tiles} field that specifies a set of "
"tiles.".format(contents=CollectionKeys.CONTENTS,
tiles=TileSetKeys.TILES))
return result