def test_utf8grid_disk_to_mbtiles():
os.mkdir('test/output')
mbtiles_to_disk('test/data/utf8grid.mbtiles', 'test/output/original', callback=None)
disk_to_mbtiles('test/output/original/', 'test/output/imported.mbtiles')
mbtiles_to_disk('test/output/imported.mbtiles', 'test/output/imported', callback=None)
assert os.path.exists('test/output/imported/0/0/0.grid.json')
original = json.load(open('test/output/original/0/0/0.grid.json'))
imported = json.load(open('test/output/imported/0/0/0.grid.json'))
assert original['data']['77'] == imported['data']['77'] == {u'ISO_A2': u'FR'}
def test_utf8grid_disk_to_mbtiles():
os.mkdir('test/output')
mbtiles_to_disk('test/data/utf8grid.mbtiles', 'test/output/original', callback=None)
disk_to_mbtiles('test/output/original/', 'test/output/imported.mbtiles')
mbtiles_to_disk('test/output/imported.mbtiles', 'test/output/imported', callback=None)
assert os.path.exists('test/output/imported/0/0/0.grid.json')
original = json.load(open('test/output/original/0/0/0.grid.json'))
imported = json.load(open('test/output/imported/0/0/0.grid.json'))
assert original['data']['77'] == imported['data']['77'] == {u'ISO_A2': u'FR'}
def __create_chart_mb_tiles(region):
region_charts_dir = os.path.join(config.unmerged_tile_dir, region + '.opt')
for chart in os.listdir(region_charts_dir):
print('archiving mbtiles for chart:', chart)
chart_dir = os.path.join(region_charts_dir, chart)
prefix = re.sub(r'\W+', '_', chart).lower()
mbtiles_file = os.path.join(config.compiled_dir, prefix + '.mbtiles')
if os.path.isfile(mbtiles_file):
os.remove(mbtiles_file)
mb.disk_to_mbtiles(chart_dir, mbtiles_file, format='png', scheme='xyz')
def test_disk_to_mbtiles_zyx():
os.mkdir('test/output')
disk_to_mbtiles('test/data/tiles/zyx',
'test/output/zyx.mbtiles',
scheme='zyx',
format='png')
mbtiles_to_disk('test/output/zyx.mbtiles',
'test/output/tiles',
callback=None)
assert os.path.exists('test/output/tiles/3/1/5.png')
def _create_region_mb_tiles(checkpoint_store, profile, region):
point = CheckPoint.CHECKPOINT_ARCHIVE
if checkpoint_store.get_checkpoint(region, profile) < point:
print('archiving mbtiles for region:', region)
region_dir = os.path.join(config.merged_tile_dir, region + '.opt')
mbtiles_file = os.path.join(config.compiled_dir, region + '.mbtiles')
if os.path.isfile(mbtiles_file):
os.remove(mbtiles_file)
mb.disk_to_mbtiles(region_dir, mbtiles_file, format='png', scheme='xyz')
checkpoint_store.clear_checkpoint(region, profile, point)
else:
print('skipping checkpoint', point)
def test_mbtiles_to_disk_and_back():
mbtiles_to_disk('test/data/one_tile.mbtiles', 'test/output')
assert os.path.exists('test/output/0/0/0.png')
disk_to_mbtiles('test/output/', 'test/output/one.mbtiles')
assert os.path.exists('test/output/one.mbtiles')
def run(self, force=False):
"""
Build a MBTile file.
force -- overwrite if MBTiles file already exists.
"""
if os.path.exists(self.filepath):
if force:
logger.warn(_("%s already exists. Overwrite.") % self.filepath)
os.remove(self.filepath)
else:
# Already built, do not do anything.
logger.info(_("%s already exists. Nothing to do.") % self.filepath)
return
# Clean previous runs
self._clean_gather()
# If no coverage added, use bottom layer metadata
if len(self._bboxes) == 0 and len(self._layers) > 0:
bottomlayer = self._layers[0]
metadata = bottomlayer.reader.metadata()
if 'bounds' in metadata:
logger.debug(_("Use bounds of bottom layer %s") % bottomlayer)
bbox = map(float, metadata.get('bounds', '').split(','))
zoomlevels = range(int(metadata.get('minzoom', 0)), int(metadata.get('maxzoom', 0)))
self.add_coverage(bbox=bbox, zoomlevels=zoomlevels)
# Compute list of tiles
tileslist = set()
for bbox, levels in self._bboxes:
logger.debug(_("Compute list of tiles for bbox %s on zooms %s.") % (bbox, levels))
bboxlist = self.tileslist(bbox, levels)
logger.debug(_("Add %s tiles.") % len(bboxlist))
tileslist = tileslist.union(bboxlist)
logger.debug(_("%s tiles in total.") % len(tileslist))
self.nbtiles = len(tileslist)
if not self.nbtiles:
raise EmptyCoverageError(_("No tiles are covered by bounding boxes : %s") % self._bboxes)
logger.debug(_("%s tiles to be packaged.") % self.nbtiles)
# Go through whole list of tiles and gather them in tmp_dir
self.rendered = 0
for (z, x, y) in tileslist:
try:
self._gather((z, x, y))
except Exception as e:
logger.warn(e)
if not self.ignore_errors:
raise
logger.debug(_("%s tiles were missing.") % self.rendered)
# Some metadata
middlezoom = self.zoomlevels[len(self.zoomlevels)/2]
lat = self.bounds[1] + (self.bounds[3] - self.bounds[1])/2
lon = self.bounds[0] + (self.bounds[2] - self.bounds[0])/2
metadata = {}
metadata['name'] = str(uuid.uuid4())
metadata['format'] = self._tile_extension[1:]
metadata['minzoom'] = self.zoomlevels[0]
metadata['maxzoom'] = self.zoomlevels[-1]
metadata['bounds'] = '%s,%s,%s,%s' % tuple(self.bounds)
metadata['center'] = '%s,%s,%s' % (lon, lat, middlezoom)
#display informations from the grids on hover
content_to_display = ''
for field_name in self.grid_fields:
content_to_display += "{{{ %s }}}<br>" % field_name
metadata['template'] = '{{#__location__}}{{/__location__}} {{#__teaser__}} \
%s {{/__teaser__}}{{#__full__}}{{/__full__}}' % content_to_display
metadatafile = os.path.join(self.tmp_dir, 'metadata.json')
with open(metadatafile, 'w') as output:
json.dump(metadata, output)
# TODO: add UTF-Grid of last layer, if any
# Package it!
logger.info(_("Build MBTiles file '%s'.") % self.filepath)
extension = self.tile_format.split("image/")[-1]
disk_to_mbtiles(
self.tmp_dir,
self.filepath,
format=extension,
scheme=self.cache.scheme
)
try:
os.remove("%s-journal" % self.filepath) # created by mbutil
except OSError, e:
pass
def run(self, force=False, metadata={}):
"""
Build a MBTile file.
force -- overwrite if MBTiles file already exists.
metadata -- a dictionary object containing metadata key/values to set
on metadata table
"""
if os.path.exists(self.filepath):
if force:
logger.warn(_("%s already exists. Overwrite.") % self.filepath)
os.remove(self.filepath)
else:
# Already built, do not do anything.
logger.info(_("%s already exists. Nothing to do.") % self.filepath)
return
# Clean previous runs
self._clean_gather()
# If no coverage added, use bottom layer metadata
if len(self._bboxes) == 0 and len(self._layers) > 0:
bottomlayer = self._layers[0]
base_metadata = bottomlayer.reader.metadata()
if 'bounds' in base_metadata:
logger.debug(_("Use bounds of bottom layer %s") % bottomlayer)
bbox = map(float, base_metadata.get('bounds', '').split(','))
zoomlevels = range(int(base_metadata.get('minzoom', 0)), int(base_metadata.get('maxzoom', 0)))
self.add_coverage(bbox=bbox, zoomlevels=zoomlevels)
# Compute list of tiles
tileslist = set()
for bbox, levels in self._bboxes:
logger.debug(_("Compute list of tiles for bbox %s on zooms %s.") % (bbox, levels))
bboxlist = self.tileslist(bbox, levels)
logger.debug(_("Add %s tiles.") % len(bboxlist))
tileslist = tileslist.union(bboxlist)
logger.debug(_("%s tiles in total.") % len(tileslist))
self.nbtiles = len(tileslist)
if not self.nbtiles:
raise EmptyCoverageError(_("No tiles are covered by bounding boxes : %s") % self._bboxes)
logger.debug(_("%s tiles to be packaged.") % self.nbtiles)
# Go through whole list of tiles and gather them in tmp_dir
self.rendered = 0
for (z, x, y) in tileslist:
try:
self._gather((z, x, y))
except Exception as e:
logger.warn(e)
if not self.ignore_errors:
raise
logger.debug(_("%s tiles were missing.") % self.rendered)
# Some metadata
middlezoom = self.zoomlevels[len(self.zoomlevels)/2]
lat = self.bounds[1] + (self.bounds[3] - self.bounds[1])/2
lon = self.bounds[0] + (self.bounds[2] - self.bounds[0])/2
out_metadata = {}
out_metadata['version'] = metadata.get("version", "1.1")
out_metadata['type'] = metadata.get("type", "baselayer")
out_metadata['description'] = metadata.get("description", "created by landez")
out_metadata['name'] = metadata.get("name", str(uuid.uuid4()))
out_metadata['format'] = self._tile_extension[1:]
out_metadata['minzoom'] = self.zoomlevels[0]
out_metadata['maxzoom'] = self.zoomlevels[-1]
out_metadata['bounds'] = '%s,%s,%s,%s' % tuple(self.bounds)
out_metadata['center'] = '%s,%s,%s' % (lon, lat, middlezoom)
#display informations from the grids on hover
content_to_display = ''
for field_name in self.grid_fields:
content_to_display += "{{{ %s }}}<br>" % field_name
out_metadata['template'] = '{{#__location__}}{{/__location__}} {{#__teaser__}} \
%s {{/__teaser__}}{{#__full__}}{{/__full__}}' % content_to_display
metadatafile = os.path.join(self.tmp_dir, 'metadata.json')
with open(metadatafile, 'w') as output:
json.dump(out_metadata, output)
# TODO: add UTF-Grid of last layer, if any
# Package it!
logger.info(_("Build MBTiles file '%s'.") % self.filepath)
extension = self.tile_format.split("image/")[-1]
disk_to_mbtiles(
self.tmp_dir,
self.filepath,
format=extension,
scheme=self.cache.scheme
)
try:
os.remove("%s-journal" % self.filepath) # created by mbutil
except OSError, e:
pass
tiles_dir,
tiles_ext,
)
# Import `tiles` directory into a `MBTiles` file
if mbtiles_path:
if os.path.isfile(mbtiles_path):
# `MBTiles` file must not already exist
sys.stderr.write("Importing tiles into already-existing MBTiles is not yet supported\n")
sys.exit(1)
metadata = {
"name": os.path.splitext(os.path.basename(mbtiles_path))[0],
"format": tiles_ext,
"bounds": f"{str(args.bbox[0])}, {str(args.bbox[1])}, {str(args.bbox[2])}, {str(args.bbox[3])}",
"minzoom": str(args.min),
"maxzoom": str(args.max),
"attribution": "",
"description": "",
"type": "",
"version": "",
}
metadata = {k: v for k, v in metadata.items() if v != ""}
with open(os.path.join(tiles_dir, "metadata.json"), "w") as outfile:
json.dump(metadata, outfile, sort_keys=True, indent=4, separators=(",", ": "))
args.format = tiles_ext
disk_to_mbtiles(tiles_dir, mbtiles_path, **args.__dict__)
v2 = kap_px_at(kap_px0[0], kap_px0[1]+1)
v3 = kap_px_at(kap_px0[0]+1, kap_px0[1]+1)
def interp(d, v0, v1):
return map(lambda v0, v1: v0*(1-d) + v1*d, v0, v1)
v01 = interp(d[0], v0, v1)
v23 = interp(d[0], v2, v3)
v = interp(d[1], v01, v23)
else:
v = kap_px_at(int(round(kap_px0[0])),
int(round(kap_px0[1])))
v = map(lambda x : min(max(int(x), 0), 255), v)
ind = 4*x
tile_pixels[y][ind+0] = v[0]
tile_pixels[y][ind+1] = v[1]
tile_pixels[y][ind+2] = v[2]
tile_pixels[y][ind+3] = 255
png.from_array(tile_pixels, 'RGBA').save(tile_path)
# write mbtiles to disk
disk_to_mbtiles(tiles_dir, sys.argv[2])
# cleanup
import shutil
shutil.rmtree(tiles_dir, ignore_errors=True)
def run(self):
logging.basicConfig(level=logging.DEBUG)
mbutil.disk_to_mbtiles(self.args[0],self.args[1],**self.args[2])
print "\nDone"
def run(self, force=False):
"""
Build a MBTile file, only if it does not exist.
"""
if os.path.exists(self.filepath):
if force:
logger.warn(_("%s already exists. Overwrite.") % self.filepath)
else:
# Already built, do not do anything.
logger.info(_("%s already exists. Nothing to do.") % self.filepath)
return
# Clean previous runs
self.clean(full=force)
# If no coverage added, use bottom layer metadata
if len(self._bboxes) == 0 and len(self._layers) > 0:
bottomlayer = self._layers[0]
metadata = bottomlayer.reader.metadata()
bbox = map(float, metadata.get('bounds', '').split(','))
zoomlevels = range(int(metadata.get('minzoom', 0)), int(metadata.get('maxzoom', 0)))
self.add_coverage(bbox=bbox, zoomlevels=zoomlevels)
# Compute list of tiles
tileslist = set()
for bbox, levels in self._bboxes:
logger.debug(_("Compute list of tiles for bbox %s on zooms %s.") % (bbox, levels))
bboxlist = self.tileslist(bbox, levels)
logger.debug(_("Add %s tiles.") % len(bboxlist))
tileslist = tileslist.union(bboxlist)
logger.debug(_("%s tiles in total.") % len(tileslist))
self.nbtiles = len(tileslist)
if not self.nbtiles:
raise EmptyCoverageError(_("No tiles are covered by bounding boxes : %s") % self._bboxes)
logger.debug(_("%s tiles to be packaged.") % self.nbtiles)
# Go through whole list of tiles and gather them in tmp_dir
self.rendered = 0
for (z, x, y) in tileslist:
self.prepare_tile((z, x, y))
logger.debug(_("%s tiles were missing.") % self.rendered)
# Some metadata
middlezoom = self.zoomlevels[len(self.zoomlevels)/2]
lat = self.bounds[1] + (self.bounds[3] - self.bounds[1])/2
lon = self.bounds[0] + (self.bounds[2] - self.bounds[0])/2
metadata = {}
metadata['minzoom'] = self.zoomlevels[0]
metadata['maxzoom'] = self.zoomlevels[-1]
metadata['bounds'] = '%s,%s,%s,%s' % tuple(self.bounds)
metadata['center'] = '%s,%s,%s' % (lon, lat, middlezoom)
metadatafile = os.path.join(self.tmp_dir, 'metadata.json')
with open(metadatafile, 'w') as output:
json.dump(metadata, output)
# TODO: add UTF-Grid of last layer, if any
# Package it!
logger.info(_("Build MBTiles file '%s'.") % self.filepath)
disk_to_mbtiles(self.tmp_dir, self.filepath)
self.clean()
def run(self, force=False):
"""
Build a MBTile file.
force -- overwrite if MBTiles file already exists.
"""
if os.path.exists(self.filepath):
if force:
logger.warn(_("%s already exists. Overwrite.") % self.filepath)
os.remove(self.filepath)
else:
# Already built, do not do anything.
logger.info(
_("%s already exists. Nothing to do.") % self.filepath)
return
# Clean previous runs
self._clean_gather()
# If no coverage added, use bottom layer metadata
if len(self._bboxes) == 0 and len(self._layers) > 0:
bottomlayer = self._layers[0]
metadata = bottomlayer.reader.metadata()
if 'bounds' in metadata:
logger.debug(_("Use bounds of bottom layer %s") % bottomlayer)
bbox = map(float, metadata.get('bounds', '').split(','))
zoomlevels = range(int(metadata.get('minzoom', 0)),
int(metadata.get('maxzoom', 0)))
self.add_coverage(bbox=bbox, zoomlevels=zoomlevels)
# Compute list of tiles
tileslist = set()
for bbox, levels in self._bboxes:
logger.debug(
_("Compute list of tiles for bbox %s on zooms %s.") %
(bbox, levels))
bboxlist = self.tileslist(bbox, levels)
logger.debug(_("Add %s tiles.") % len(bboxlist))
tileslist = tileslist.union(bboxlist)
logger.debug(_("%s tiles in total.") % len(tileslist))
self.nbtiles = len(tileslist)
if not self.nbtiles:
raise EmptyCoverageError(
_("No tiles are covered by bounding boxes : %s") %
self._bboxes)
logger.debug(_("%s tiles to be packaged.") % self.nbtiles)
# Go through whole list of tiles and gather them in tmp_dir
self.rendered = 0
for (z, x, y) in tileslist:
self._gather((z, x, y))
logger.debug(_("%s tiles were missing.") % self.rendered)
# Some metadata
middlezoom = self.zoomlevels[len(self.zoomlevels) / 2]
lat = self.bounds[1] + (self.bounds[3] - self.bounds[1]) / 2
lon = self.bounds[0] + (self.bounds[2] - self.bounds[0]) / 2
metadata = {}
metadata['format'] = self._tile_extension[1:]
metadata['minzoom'] = self.zoomlevels[0]
metadata['maxzoom'] = self.zoomlevels[-1]
metadata['bounds'] = '%s,%s,%s,%s' % tuple(self.bounds)
metadata['center'] = '%s,%s,%s' % (lon, lat, middlezoom)
metadatafile = os.path.join(self.tmp_dir, 'metadata.json')
with open(metadatafile, 'w') as output:
json.dump(metadata, output)
# TODO: add UTF-Grid of last layer, if any
# Package it!
logger.info(_("Build MBTiles file '%s'.") % self.filepath)
disk_to_mbtiles(self.tmp_dir, self.filepath)
try:
os.remove("%s-journal" % self.filepath) # created by mbutil
except OSError, e:
logger.debug(e)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
parser = OptionParser(usage="usage: %prog [options] input output")
parser.add_option('-w', '--window', dest='window',
help='compression window size. larger values faster, dangerouser',
type='int',
default=2000)
(options, args) = parser.parse_args()
# Transfer operations
if len(args) == 2:
if os.path.isfile(args[0]) and os.path.exists(args[1]):
sys.stderr.write('To export MBTiles to disk, specify a directory that does not yet exist\n')
sys.exit(1)
if os.path.isfile(args[0]) and not os.path.exists(args[1]):
mbtiles_file, directory_path = args
mbtiles_to_disk(mbtiles_file, directory_path)
if os.path.isdir(args[0]) and os.path.isfile(args[1]):
sys.stderr.write('Importing tiles into already-existing MBTiles is not yet supported\n')
sys.exit(1)
if os.path.isdir(args[0]) and not os.path.isfile(args[0]):
directory_path, mbtiles_file = args
disk_to_mbtiles(directory_path, mbtiles_file)
else:
parser.print_help()
def run(self, force=False):
"""
Build a MBTile file.
force -- overwrite if MBTiles file already exists.
"""
if os.path.exists(self.filepath):
if force:
logger.warn(_("%s already exists. Overwrite.") % self.filepath)
os.remove(self.filepath)
else:
# Already built, do not do anything.
logger.info(_("%s already exists. Nothing to do.") % self.filepath)
return
# Clean previous runs
self._clean_gather()
# If no coverage added, use bottom layer metadata
if len(self._bboxes) == 0 and len(self._layers) > 0:
bottomlayer = self._layers[0]
metadata = bottomlayer.reader.metadata()
if 'bounds' in metadata:
logger.debug(_("Use bounds of bottom layer %s") % bottomlayer)
bbox = map(float, metadata.get('bounds', '').split(','))
zoomlevels = range(int(metadata.get('minzoom', 0)), int(metadata.get('maxzoom', 0)))
self.add_coverage(bbox=bbox, zoomlevels=zoomlevels)
# Compute list of tiles
tileslist = set()
for bbox, levels in self._bboxes:
logger.debug(_("Compute list of tiles for bbox %s on zooms %s.") % (bbox, levels))
bboxlist = self.tileslist(bbox, levels)
logger.debug(_("Add %s tiles.") % len(bboxlist))
tileslist = tileslist.union(bboxlist)
logger.debug(_("%s tiles in total.") % len(tileslist))
self.nbtiles = len(tileslist)
if not self.nbtiles:
raise EmptyCoverageError(_("No tiles are covered by bounding boxes : %s") % self._bboxes)
logger.debug(_("%s tiles to be packaged.") % self.nbtiles)
# Go through whole list of tiles and gather them in tmp_dir
self.rendered = 0
for (z, x, y) in tileslist:
self._gather((z, x, y))
logger.debug(_("%s tiles were missing.") % self.rendered)
# Some metadata
middlezoom = self.zoomlevels[len(self.zoomlevels)/2]
lat = self.bounds[1] + (self.bounds[3] - self.bounds[1])/2
lon = self.bounds[0] + (self.bounds[2] - self.bounds[0])/2
metadata = {}
metadata['format'] = self._tile_extension[1:]
metadata['minzoom'] = self.zoomlevels[0]
metadata['maxzoom'] = self.zoomlevels[-1]
metadata['bounds'] = '%s,%s,%s,%s' % tuple(self.bounds)
metadata['center'] = '%s,%s,%s' % (lon, lat, middlezoom)
metadata['version'] = '1.0.0'
metadatafile = os.path.join(self.tmp_dir, 'metadata.json')
with open(metadatafile, 'w') as output:
json.dump(metadata, output)
# TODO: add UTF-Grid of last layer, if any
logger.info(_("Optimise PNG images"))
if find_executable('optipng'):
png_optimize(self.tmp_dir)
# Package it!
logger.info(_("Build MBTiles file '%s'.") % self.filepath)
disk_to_mbtiles(self.tmp_dir, self.filepath)
try:
os.remove("%s-journal" % self.filepath) # created by mbutil
except OSError, e:
logger.debug(e)
def test_disk_to_mbtiles_zyx():
os.mkdir('test/output')
disk_to_mbtiles('test/data/tiles/zyx', 'test/output/zyx.mbtiles', scheme='zyx', format='png')
mbtiles_to_disk('test/output/zyx.mbtiles', 'test/output/tiles', callback=None)
assert os.path.exists('test/output/tiles/3/1/5.png')
