def _tile_location_tc(self, tile, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> c = FileCache(cache_dir='/tmp/cache/', file_ext='png')
>>> c.tile_location(Tile((3, 4, 2))).replace('\\\\', '/')
'/tmp/cache/02/000/000/003/000/000/004.png'
"""
if tile.location is None:
x, y, z = tile.coord
parts = (self.level_location(z),
"%03d" % int(x / 1000000),
"%03d" % (int(x / 1000) % 1000),
"%03d" % (int(x) % 1000),
"%03d" % int(y / 1000000),
"%03d" % (int(y / 1000) % 1000),
"%03d.%s" % (int(y) % 1000, self.file_ext))
tile.location = os.path.join(*parts)
if create_dir:
ensure_directory(tile.location)
return tile.location
def tile_location_quadkey(tile, cache_dir, file_ext, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> tile_location_quadkey(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
'/tmp/cache/11.png'
"""
if tile.location is None:
x, y, z = tile.coord
quadKey = ""
for i in range(z,0,-1):
digit = 0
mask = 1 << (i-1)
if (x & mask) != 0:
digit += 1
if (y & mask) != 0:
digit += 2
quadKey += str(digit)
tile.location = os.path.join(
cache_dir, quadKey + '.' + file_ext
)
if create_dir:
ensure_directory(tile.location)
return tile.location
def do_POST(self):
length = int(self.headers['content-length'])
json_data = self.rfile.read(length)
task = json.loads(json_data)
eq_(task['command'], 'tile')
# request main tile of metatile
eq_(task['tiles'], [[15, 17, 5]])
eq_(task['cache_identifier'], 'wms_cache_GLOBAL_MERCATOR')
eq_(task['priority'], 100)
# this id should not change for the same tile/cache_identifier combination
eq_(task['id'], 'aeb52b506e4e82d0a1edf649d56e0451cfd5862c')
# manually create tile renderd should create
tile_filename = os.path.join(
test_self.config['cache_dir'],
'wms_cache_EPSG900913/05/000/000/016/000/000/016.jpeg')
ensure_directory(tile_filename)
with open(tile_filename, 'w') as f:
f.write(
create_tmp_image((256, 256),
format='jpeg',
color=(255, 0, 100)))
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write('{"status": "ok"}')
def tile_location_mp(tile, cache_dir, file_ext, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> tile_location_mp(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
'/tmp/cache/02/0000/0003/0000/0004.png'
>>> tile_location_mp(Tile((12345678, 98765432, 22)), '/tmp/cache', 'png').replace('\\\\', '/')
'/tmp/cache/22/1234/5678/9876/5432.png'
"""
if tile.location is None:
x, y, z = tile.coord
parts = (cache_dir,
level_part(z),
"%04d" % int(x / 10000),
"%04d" % (int(x) % 10000),
"%04d" % int(y / 10000),
"%04d.%s" % (int(y) % 10000, file_ext))
tile.location = os.path.join(*parts)
if create_dir:
ensure_directory(tile.location)
return tile.location
def _tile_location_quadkey(self, tile, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> from mapproxy.cache.file import FileCache
>>> c = FileCache(cache_dir='/tmp/cache/', file_ext='png', directory_layout='quadkey')
>>> c.tile_location(Tile((3, 4, 2))).replace('\\\\', '/')
'/tmp/cache/11.png'
"""
if tile.location is None:
x, y, z = tile.coord
quadKey = ""
for i in range(z, 0, -1):
digit = 0
mask = 1 << (i - 1)
if (x & mask) != 0:
digit += 1
if (y & mask) != 0:
digit += 2
quadKey += str(digit)
tile.location = os.path.join(self.cache_dir, quadKey + "." + self.file_ext)
if create_dir:
ensure_directory(tile.location)
return tile.location
def tile_location_mp(tile, cache_dir, file_ext, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> tile_location_mp(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
'/tmp/cache/02/0000/0003/0000/0004.png'
>>> tile_location_mp(Tile((12345678, 98765432, 22)), '/tmp/cache', 'png').replace('\\\\', '/')
'/tmp/cache/22/1234/5678/9876/5432.png'
"""
if tile.location is None:
x, y, z = tile.coord
parts = (cache_dir,
level_part(z),
"%04d" % int(x / 10000),
"%04d" % (int(x) % 10000),
"%04d" % int(y / 10000),
"%04d.%s" % (int(y) % 10000, file_ext))
tile.location = os.path.join(*parts)
if create_dir:
ensure_directory(tile.location)
return tile.location
def _tile_location_tc(self, tile, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> c = FileCache(cache_dir='/tmp/cache/', file_ext='png')
>>> c.tile_location(Tile((3, 4, 2))).replace('\\\\', '/')
'/tmp/cache/02/000/000/003/000/000/004.png'
"""
if tile.location is None:
x, y, z = tile.coord
parts = (
self._level_location(z),
"%03d" % int(x / 1000000),
"%03d" % (int(x / 1000) % 1000),
"%03d" % (int(x) % 1000),
"%03d" % int(y / 1000000),
"%03d" % (int(y / 1000) % 1000),
"%03d.%s" % (int(y) % 1000, self.file_ext),
)
tile.location = os.path.join(*parts)
if create_dir:
ensure_directory(tile.location)
return tile.location
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(self.mbtile_file + '.init.lck',
remove_on_unlock=REMOVE_ON_UNLOCK):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(os.path.join(os.path.dirname(self.mbtile_file), 'init.lck'),
remove_on_unlock=True):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def _init_bundle(self):
log.info("Init Bundle %s" % self.filename)
ensure_directory(self.filename)
write_atomic(self.filename, struct.pack(*[self.BUNDLE_BYTEORDER + self.BUNDLE_HEADER_FORMAT + self.BUNDLE_INDEX_FORMAT] +
list(self.header.values()) +
self.index # empty index
))
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(os.path.join(self.lock_dir, 'init.lck'),
timeout=self.lock_timeout,
remove_on_unlock=True):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(os.path.join(self.lock_dir, 'init.lck'),
timeout=self.lock_timeout,
remove_on_unlock=True):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def ensure_mbtile(self):
if not os.path.exists(self.mbtile_file):
with FileLock(os.path.join(os.path.dirname(self.mbtile_file),
'init.lck'),
remove_on_unlock=True):
if not os.path.exists(self.mbtile_file):
ensure_directory(self.mbtile_file)
self._initialize_mbtile()
def _init_bundle(self):
ensure_directory(self.filename)
header = list(BUNDLE_HEADER)
header[13], header[11] = self.tile_offsets
header[14], header[12] = header[13]+127, header[11]+127
write_atomic(self.filename,
struct.pack(BUNDLE_HEADER_STRUCT_FORMAT, *header) +
# zero-size entry for each tile
(b'\x00' * (BUNDLEX_GRID_HEIGHT * BUNDLEX_GRID_WIDTH * 4)))
def _init_bundle(self):
ensure_directory(self.filename)
header = list(BUNDLE_V1_HEADER)
header[10], header[8] = self.tile_offsets
header[11], header[9] = header[10]+127, header[8]+127
write_atomic(self.filename,
struct.pack(BUNDLE_V1_HEADER_STRUCT_FORMAT, *header) +
# zero-size entry for each tile
(b'\x00' * (BUNDLEX_V1_GRID_HEIGHT * BUNDLEX_V1_GRID_WIDTH * 4)))
def _init_index(self):
self._initialized = True
if os.path.exists(self.filename):
return
ensure_directory(self.filename)
buf = BytesIO()
buf.write(struct.pack(BUNDLE_V2_HEADER_STRUCT_FORMAT, *BUNDLE_V2_HEADER))
# Empty index (ArcGIS stores an offset of 4 and size of 0 for missing tiles)
buf.write(struct.pack('<%dQ' % BUNDLE_V2_TILES, *(4, ) * BUNDLE_V2_TILES))
write_atomic(self.filename, buf.getvalue())
def ensure_gpkg(self):
if not os.path.isfile(self.geopackage_file):
with FileLock(self.geopackage_file + '.init.lck',
remove_on_unlock=REMOVE_ON_UNLOCK):
ensure_directory(self.geopackage_file)
self._initialize_gpkg()
else:
if not self.check_gpkg():
ensure_directory(self.geopackage_file)
self._initialize_gpkg()
def ensure_gpkg(self):
if not os.path.isfile(self.geopackage_file):
with FileLock(self.geopackage_file + '.init.lck',
remove_on_unlock=REMOVE_ON_UNLOCK):
ensure_directory(self.geopackage_file)
self._initialize_gpkg()
else:
if not self.check_gpkg():
ensure_directory(self.geopackage_file)
self._initialize_gpkg()
def _init_index(self):
self._initialized = True
if os.path.exists(self.filename):
return
ensure_directory(self.filename)
buf = BytesIO()
buf.write(BUNDLEX_HEADER)
for i in range(BUNDLEX_GRID_WIDTH * BUNDLEX_GRID_HEIGHT):
buf.write(struct.pack('<Q', (i*4)+BUNDLE_HEADER_SIZE)[:5])
buf.write(BUNDLEX_FOOTER)
write_atomic(self.filename, buf.getvalue())
def _single_color_tile_location(self, color, create_dir=False):
"""
>>> c = FileCache(cache_dir='/tmp/cache/', file_ext='png')
>>> c._single_color_tile_location((254, 0, 4)).replace('\\\\', '/')
'/tmp/cache/single_color_tiles/fe0004.png'
"""
parts = (self.cache_dir, "single_color_tiles", "".join("%02x" % v for v in color) + "." + self.file_ext)
location = os.path.join(*parts)
if create_dir:
ensure_directory(location)
return location
def _init_index(self):
self._initialized = True
if os.path.exists(self.filename):
return
ensure_directory(self.filename)
buf = BytesIO()
buf.write(BUNDLEX_HEADER)
for i in range(BUNDLEX_GRID_WIDTH * BUNDLEX_GRID_HEIGHT):
buf.write(struct.pack('<Q', (i*4)+BUNDLE_HEADER_SIZE)[:5])
buf.write(BUNDLEX_FOOTER)
write_atomic(self.filename, buf.getvalue())
def _single_color_tile_location(self, color, create_dir=False):
"""
>>> c = FileCache(cache_dir='/tmp/cache/', file_ext='png')
>>> c._single_color_tile_location((254, 0, 4)).replace('\\\\', '/')
'/tmp/cache/single_color_tiles/fe0004.png'
"""
parts = (self.cache_dir, 'single_color_tiles',
''.join('%02x' % v for v in color) + '.' + self.file_ext)
location = os.path.join(*parts)
if create_dir:
ensure_directory(location)
return location
def _init_index(self):
self._initialized = True
if os.path.exists(self.filename):
return
ensure_directory(self.filename)
buf = BytesIO()
buf.write(
struct.pack(BUNDLE_V2_HEADER_STRUCT_FORMAT, *BUNDLE_V2_HEADER))
# Empty index (ArcGIS stores an offset of 4 and size of 0 for missing tiles)
buf.write(
struct.pack('<%dQ' % BUNDLE_V2_TILES,
*(4, ) * BUNDLE_V2_TILES))
write_atomic(self.filename, buf.getvalue())
def make_tile(self, coord=(0, 0, 0), timestamp=None):
"""
Create file for tile at `coord` with given timestamp.
"""
tile_dir = os.path.join(self.dir, 'cache/one_EPSG4326/%02d/000/000/%03d/000/000/' %
(coord[2], coord[0]))
ensure_directory(tile_dir)
tile = os.path.join(tile_dir + '%03d.png' % coord[1])
open(tile, 'wb').write(b'')
if timestamp:
os.utime(tile, (timestamp, timestamp))
return tile
def store(self, legend):
if legend.stored:
return
if legend.location is None:
hash = legend_hash(legend.id, legend.scale)
legend.location = os.path.join(self.cache_dir, hash) + '.' + self.file_ext
ensure_directory(legend.location)
data = legend.source.as_buffer(ImageOptions(format='image/' + self.file_ext), seekable=True)
data.seek(0)
log.debug('writing to %s' % (legend.location))
write_atomic(legend.location, data.read())
data.seek(0)
legend.stored = True
def store(self, legend):
if legend.stored:
return
if legend.location is None:
hash = legend_hash(legend.id, legend.scale)
legend.location = os.path.join(self.cache_dir, hash) + '.' + self.file_ext
ensure_directory(legend.location)
data = legend.source.as_buffer(ImageOptions(format='image/' + self.file_ext), seekable=True)
data.seek(0)
log.debug('writing to %s' % (legend.location))
write_atomic(legend.location, data.read())
data.seek(0)
legend.stored = True
def tile_location_arcgiscache(tile, cache_dir, file_ext, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> tile_location_arcgiscache(Tile((1234567, 87654321, 9)), '/tmp/cache', 'png').replace('\\\\', '/')
'/tmp/cache/L09/R05397fb1/C0012d687.png'
"""
if tile.location is None:
x, y, z = tile.coord
parts = (cache_dir, 'L%02d' % z, 'R%08x' % y, 'C%08x.%s' % (x, file_ext))
tile.location = os.path.join(*parts)
if create_dir:
ensure_directory(tile.location)
return tile.location
def tile_location_reverse_tms(tile, cache_dir, file_ext, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> tile_location_reverse_tms(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
'/tmp/cache/4/3/2.png'
"""
if tile.location is None:
x, y, z = tile.coord
tile.location = os.path.join(cache_dir, str(y), str(x),
str(z) + '.' + file_ext)
if create_dir:
ensure_directory(tile.location)
return tile.location
def tile_location_arcgiscache(tile, cache_dir, file_ext, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> tile_location_arcgiscache(Tile((1234567, 87654321, 9)), '/tmp/cache', 'png').replace('\\\\', '/')
'/tmp/cache/L09/R05397fb1/C0012d687.png'
"""
if tile.location is None:
x, y, z = tile.coord
parts = (cache_dir, 'L%02d' % z, 'R%08x' % y, 'C%08x.%s' % (x, file_ext))
tile.location = os.path.join(*parts)
if create_dir:
ensure_directory(tile.location)
return tile.location
def _tile_location_tms(self, tile, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> c = FileCache(cache_dir='/tmp/cache/', file_ext='png', directory_layout='tms')
>>> c.tile_location(Tile((3, 4, 2))).replace('\\\\', '/')
'/tmp/cache/2/3/4.png'
"""
if tile.location is None:
x, y, z = tile.coord
tile.location = os.path.join(self.level_location(str(z)), str(x), str(y) + "." + self.file_ext)
if create_dir:
ensure_directory(tile.location)
return tile.location
def tile_location_reverse_tms(tile, cache_dir, file_ext, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> tile_location_reverse_tms(Tile((3, 4, 2)), '/tmp/cache', 'png').replace('\\\\', '/')
'/tmp/cache/4/3/2.png'
"""
if tile.location is None:
x, y, z = tile.coord
tile.location = os.path.join(
cache_dir, str(y), str(x), str(z) + '.' + file_ext
)
if create_dir:
ensure_directory(tile.location)
return tile.location
def _tile_location_tms(self, tile, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> c = FileCache(cache_dir='/tmp/cache/', file_ext='png', directory_layout='tms')
>>> c.tile_location(Tile((3, 4, 2))).replace('\\\\', '/')
'/tmp/cache/2/3/4.png'
"""
if tile.location is None:
x, y, z = tile.coord
tile.location = os.path.join(self.level_location(str(z)), str(x),
str(y) + '.' + self.file_ext)
if create_dir:
ensure_directory(tile.location)
return tile.location
def do_POST(self):
length = int(self.headers['content-length'])
json_data = self.rfile.read(length)
task = json.loads(json_data.decode('utf-8'))
eq_(task['command'], 'tile')
eq_(task['tiles'], [[10, 20, 6]])
eq_(task['cache_identifier'], 'tms_cache_GLOBAL_MERCATOR')
eq_(task['priority'], 100)
# this id should not change for the same tile/cache_identifier combination
eq_(task['id'], 'cf35c1c927158e188d8fbe0db380c1772b536da9')
# manually create tile renderd should create
tile_filename = os.path.join(test_self.config['cache_dir'],
'tms_cache_EPSG900913/06/000/000/010/000/000/020.png')
ensure_directory(tile_filename)
with open(tile_filename, 'wb') as f:
f.write(b"foobaz")
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write(b'{"status": "ok"}')
def _tile_location_arcgiscache(self, tile, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> from mapproxy.cache.file import FileCache
>>> c = FileCache(cache_dir='/tmp/cache/', file_ext='png', directory_layout='arcgis')
>>> c.tile_location(Tile((1234567, 87654321, 9))).replace('\\\\', '/')
'/tmp/cache/L09/R05397fb1/C0012d687.png'
"""
if tile.location is None:
x, y, z = tile.coord
parts = (self._level_location_arcgiscache(z), "R%08x" % y, "C%08x.%s" % (x, self.file_ext))
tile.location = os.path.join(*parts)
if create_dir:
ensure_directory(tile.location)
return tile.location
def do_POST(self):
length = int(self.headers['content-length'])
json_data = self.rfile.read(length)
task = json.loads(json_data.decode('utf-8'))
eq_(task['command'], 'tile')
# request main tile of metatile
eq_(task['tiles'], [[15, 17, 5]])
eq_(task['cache_identifier'], 'wms_cache_GLOBAL_MERCATOR')
eq_(task['priority'], 100)
# this id should not change for the same tile/cache_identifier combination
eq_(task['id'], 'aeb52b506e4e82d0a1edf649d56e0451cfd5862c')
# manually create tile renderd should create
tile_filename = os.path.join(test_self.config['cache_dir'],
'wms_cache_EPSG900913/05/000/000/016/000/000/016.jpeg')
ensure_directory(tile_filename)
with open(tile_filename, 'wb') as f:
f.write(create_tmp_image((256, 256), format='jpeg', color=(255, 0, 100)))
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write(b'{"status": "ok"}')
def _tile_location_arcgiscache(self, tile, create_dir=False):
"""
Return the location of the `tile`. Caches the result as ``location``
property of the `tile`.
:param tile: the tile object
:param create_dir: if True, create all necessary directories
:return: the full filename of the tile
>>> from mapproxy.cache.tile import Tile
>>> from mapproxy.cache.file import FileCache
>>> c = FileCache(cache_dir='/tmp/cache/', file_ext='png', directory_layout='arcgis')
>>> c.tile_location(Tile((1234567, 87654321, 9))).replace('\\\\', '/')
'/tmp/cache/L09/R05397fb1/C0012d687.png'
"""
if tile.location is None:
x, y, z = tile.coord
parts = (self._level_location_arcgiscache(z), 'R%08x' % y,
'C%08x.%s' % (x, self.file_ext))
tile.location = os.path.join(*parts)
if create_dir:
ensure_directory(tile.location)
return tile.location