def test_png(self):
layout = WMSTileLayout(url='http://example.com/folder',
layers='l1,l2',
srs='EPSG:1000',
format='image/png',
tilegrid=self.tilegrid)
result = urlparse(layout.filename(TileCoord(0, 0, 0)))
self.assertEqual(result.netloc, 'example.com')
self.assertEqual(result.path, '/folder')
query = parse_qs(result.query)
self.assertEqual(query['LAYERS'], ['l1,l2'])
self.assertEqual(query['FORMAT'], ['image/png'])
self.assertEqual(query['TRANSPARENT'], ['TRUE'])
self.assertEqual(query['SERVICE'], ['WMS'])
self.assertEqual(query['VERSION'], ['1.1.1'])
self.assertEqual(query['REQUEST'], ['GetMap'])
self.assertEqual(query['SRS'], ['EPSG:1000'])
self.assertEqual(query['WIDTH'], ['100'])
self.assertEqual(query['HEIGHT'], ['100'])
bbox = [float(i) for i in query['BBOX'][0].split(',')]
self.assertEqual(len(bbox), 4)
self.assertEqual(bbox[0], 420000.0)
self.assertEqual(bbox[1], 340000.0)
self.assertEqual(bbox[2], 430000.0)
self.assertEqual(bbox[3], 350000.0)
def test_params(self):
layout = WMSTileLayout(
url='http://example.com/folder',
layers='l1,l2',
srs='EPSG:1000',
format='image/png',
tilegrid=self.tilegrid,
params={
'TRANSPARENT': 'FALSE',
'PARAM': 'Value',
'FILTER': 'l1:"field" = '
'{PARAM}'
''
},
)
result = urlparse(layout.filename(TileCoord(0, 0, 0)))
assert result.netloc == 'example.com'
assert result.path == '/folder'
query = parse_qs(result.query)
assert query['PARAM'] == ['Value']
assert query['FILTER'] == ['l1:"field" = ' 'Value' '']
assert query['LAYERS'] == ['l1,l2']
assert query['FORMAT'] == ['image/png']
assert query['TRANSPARENT'] == ['FALSE']
assert query['SERVICE'] == ['WMS']
assert query['VERSION'] == ['1.1.1']
assert query['REQUEST'] == ['GetMap']
assert query['SRS'] == ['EPSG:1000']
assert query['WIDTH'] == ['100']
assert query['HEIGHT'] == ['100']
bbox = [float(i) for i in query['BBOX'][0].split(',')]
assert bbox == [420000.0, 340000.0, 430000.0, 350000.0]
def test_png(self):
layout = WMSTileLayout(
url='http://example.com/folder',
layers='l1,l2',
srs='EPSG:1000',
format='image/png',
tilegrid=self.tilegrid)
result = urlparse(layout.filename(TileCoord(0, 0, 0)))
self.assertEqual(result.netloc, 'example.com')
self.assertEqual(result.path, '/folder')
query = parse_qs(result.query)
self.assertEqual(query['LAYERS'], ['l1,l2'])
self.assertEqual(query['FORMAT'], ['image/png'])
self.assertEqual(query['TRANSPARENT'], ['TRUE'])
self.assertEqual(query['SERVICE'], ['WMS'])
self.assertEqual(query['VERSION'], ['1.1.1'])
self.assertEqual(query['REQUEST'], ['GetMap'])
self.assertEqual(query['SRS'], ['EPSG:1000'])
self.assertEqual(query['WIDTH'], ['100'])
self.assertEqual(query['HEIGHT'], ['100'])
bbox = [float(i) for i in query['BBOX'][0].split(',')]
self.assertEqual(len(bbox), 4)
self.assertEqual(bbox[0], 420000.0)
self.assertEqual(bbox[1], 340000.0)
self.assertEqual(bbox[2], 430000.0)
self.assertEqual(bbox[3], 350000.0)
def test_params(self):
layout = WMSTileLayout(
url="http://example.com/folder",
layers="l1,l2",
srs="EPSG:1000",
format="image/png",
tilegrid=self.tilegrid,
params={
"TRANSPARENT": "FALSE",
"PARAM": "Value",
"FILTER": 'l1:"field" = '
"{PARAM}"
""
},
)
result = urlparse(layout.filename(TileCoord(0, 0, 0)))
assert result.netloc == "example.com"
assert result.path == "/folder"
query = parse_qs(result.query)
assert query["PARAM"] == ["Value"]
assert query["FILTER"] == ['l1:"field" = ' "Value" ""]
assert query["LAYERS"] == ["l1,l2"]
assert query["FORMAT"] == ["image/png"]
assert query["TRANSPARENT"] == ["FALSE"]
assert query["SERVICE"] == ["WMS"]
assert query["VERSION"] == ["1.1.1"]
assert query["REQUEST"] == ["GetMap"]
assert query["SRS"] == ["EPSG:1000"]
assert query["WIDTH"] == ["100"]
assert query["HEIGHT"] == ["100"]
bbox = [float(i) for i in query["BBOX"][0].split(",")]
assert bbox == [420000.0, 340000.0, 430000.0, 350000.0]
def test_border(self):
layout = WMSTileLayout(
url="http://example.com/folder",
layers="l1,l2",
srs="EPSG:1000",
format="image/png",
tilegrid=self.tilegrid,
border=10,
)
result = urlparse(layout.filename(TileCoord(0, 0, 0)))
assert result.netloc == "example.com"
assert result.path == "/folder"
query = parse_qs(result.query)
assert query["LAYERS"] == ["l1,l2"]
assert query["FORMAT"] == ["image/png"]
assert query["TRANSPARENT"] == ["TRUE"]
assert query["SERVICE"] == ["WMS"]
assert query["VERSION"] == ["1.1.1"]
assert query["REQUEST"] == ["GetMap"]
assert query["SRS"] == ["EPSG:1000"]
assert query["WIDTH"] == ["120"]
assert query["HEIGHT"] == ["120"]
bbox = [float(i) for i in query["BBOX"][0].split(",")]
assert len(bbox) == 4
assert bbox == [419000.0, 339000.0, 431000.0, 351000.0]
def test_params(self):
layout = WMSTileLayout(
url='http://example.com/folder',
layers='l1,l2',
srs='EPSG:1000',
format='image/png',
tilegrid=self.tilegrid,
params={
'TRANSPARENT': 'FALSE',
'PARAM': 'Value',
'FILTER': 'l1:"field" = ''{PARAM}'''
},
)
result = urlparse(layout.filename(TileCoord(0, 0, 0)))
assert result.netloc == 'example.com'
assert result.path == '/folder'
query = parse_qs(result.query)
assert query['PARAM'] == ['Value']
assert query['FILTER'] == ['l1:"field" = ''Value''']
assert query['LAYERS'] == ['l1,l2']
assert query['FORMAT'] == ['image/png']
assert query['TRANSPARENT'] == ['FALSE']
assert query['SERVICE'] == ['WMS']
assert query['VERSION'] == ['1.1.1']
assert query['REQUEST'] == ['GetMap']
assert query['SRS'] == ['EPSG:1000']
assert query['WIDTH'] == ['100']
assert query['HEIGHT'] == ['100']
bbox = [float(i) for i in query['BBOX'][0].split(',')]
assert bbox == [420000.0, 340000.0, 430000.0, 350000.0]
def _get_tilestore_for_layer(self, layer, gene):
if layer['type'] == 'wms':
params = layer['params'].copy()
if 'STYLES' not in params:
params['STYLES'] = ','.join(
layer['wmts_style'] for l in layer['layers'].split(','))
if layer['generate_salt']:
params['SALT'] = str(random.randint(0, 999999))
params.update(self.dimensions)
# Get the metatile image from the WMS server
return URLTileStore(
tilelayouts=(WMSTileLayout(
url=layer['url'],
layers=layer['layers'],
srs=layer['grid_ref']['srs'],
format=layer['mime_type'],
border=layer['meta_buffer'] if layer.get('meta') else 0,
tilegrid=gene.get_grid(layer['grid'])['obj'],
params=params,
), ),
headers=layer['headers'],
)
elif layer['type'] == 'mapnik': # pragma: no cover
try:
from tilecloud.store.mapnik_ import MapnikTileStore
from tilecloud_chain.mapnik_ import MapnikDropActionTileStore
except ImportError:
if 'TRAVIS' not in os.environ: # pragma nocover
logger.error("Mapnik is not available")
return None
grid = gene.get_grid(layer['grid'])
if layer['output_format'] == 'grid':
return MapnikDropActionTileStore(
tilegrid=grid['obj'],
mapfile=layer['mapfile'],
image_buffer=layer['meta_buffer']
if layer.get('meta') else 0,
data_buffer=layer['data_buffer'],
output_format=layer['output_format'],
resolution=layer['resolution'],
layers_fields=layer['layers_fields'],
drop_empty_utfgrid=layer['drop_empty_utfgrid'],
store=self.cache_tilestore,
queue_store=self.sqs_tilestore,
count=[self.count_tiles, self.count_tiles_dropped],
proj4_literal=grid['proj4_literal'],
)
else:
return MapnikTileStore(
tilegrid=grid['obj'],
mapfile=layer['mapfile'],
image_buffer=layer['meta_buffer']
if layer.get('meta') else 0,
data_buffer=layer['data_buffer'],
output_format=layer['output_format'],
proj4_literal=grid['proj4_literal'],
)
def test_metatile(self):
layout = WMSTileLayout(url='http://example.com/folder',
layers='l1,l2',
srs='EPSG:1000',
format='image/png',
tilegrid=self.tilegrid)
result = urlparse(layout.filename(TileCoord(1, 0, 0, 2)))
assert result.netloc == 'example.com'
assert result.path == '/folder'
query = parse_qs(result.query)
assert query['LAYERS'] == ['l1,l2']
assert query['FORMAT'] == ['image/png']
assert query['TRANSPARENT'] == ['TRUE']
assert query['SERVICE'] == ['WMS']
assert query['VERSION'] == ['1.1.1']
assert query['REQUEST'] == ['GetMap']
assert query['SRS'] == ['EPSG:1000']
assert query['WIDTH'] == ['200']
assert query['HEIGHT'] == ['200']
bbox = [float(i) for i in query['BBOX'][0].split(',')]
assert bbox == [420000.0, 340000.0, 430000.0, 350000.0]
def test_metatile(self):
layout = WMSTileLayout(
url='http://example.com/folder',
layers='l1,l2',
srs='EPSG:1000',
format='image/png',
tilegrid=self.tilegrid)
result = urlparse(layout.filename(TileCoord(1, 0, 0, 2)))
assert result.netloc == 'example.com'
assert result.path == '/folder'
query = parse_qs(result.query)
assert query['LAYERS'] == ['l1,l2']
assert query['FORMAT'] == ['image/png']
assert query['TRANSPARENT'] == ['TRUE']
assert query['SERVICE'] == ['WMS']
assert query['VERSION'] == ['1.1.1']
assert query['REQUEST'] == ['GetMap']
assert query['SRS'] == ['EPSG:1000']
assert query['WIDTH'] == ['200']
assert query['HEIGHT'] == ['200']
bbox = [float(i) for i in query['BBOX'][0].split(',')]
assert bbox == [420000.0, 340000.0, 430000.0, 350000.0]
def __call__(self, config_file: str, layer_name: str) -> Optional[TileStore]:
config = self.gene._gene.get_config(config_file)
layer = config.config["layers"][layer_name]
if layer["type"] == "wms":
params = layer.get("params", {}).copy()
if "STYLES" not in params:
params["STYLES"] = ",".join(layer["wmts_style"] for _ in layer["layers"].split(","))
if layer.get("generate_salt", False):
params["SALT"] = str(random.randint(0, 999999)) # nosec
# Get the metatile image from the WMS server
return URLTileStore(
tilelayouts=(
WMSTileLayout(
url=layer["url"],
layers=layer["layers"],
srs=config.config["grids"][layer["grid"]]["srs"],
format=layer["mime_type"],
border=layer["meta_buffer"] if layer["meta"] else 0,
tilegrid=self.gene._gene.get_grid(config, layer["grid"]),
params=params,
),
),
headers=layer["headers"],
)
elif layer["type"] == "mapnik":
try:
from tilecloud.store.mapnik_ import MapnikTileStore # pylint: disable=import-outside-toplevel
from tilecloud_chain.mapnik_ import ( # pylint: disable=import-outside-toplevel
MapnikDropActionTileStore,
)
except ImportError:
if os.environ.get("CI", "FALSE") == "FALSE": # pragma nocover
logger.error("Mapnik is not available", exc_info=True)
return None
grid = config.config["grids"][layer["grid"]]
if cast(str, layer.get("output_format", "png")) == "grid":
assert self.gene._count_tiles
assert self.gene._count_tiles_dropped
return MapnikDropActionTileStore(
tilegrid=self.gene._gene.get_grid(config, layer["grid"]),
mapfile=layer["mapfile"],
image_buffer=layer["meta_buffer"] if layer.get("meta") else 0,
data_buffer=layer.get("data_buffer", 128),
output_format=layer.get("output_format", "png"),
resolution=layer.get("resolution", 4),
layers_fields=layer.get("layers_fields", {}),
drop_empty_utfgrid=layer.get("drop_empty_utfgrid", False),
store=self.gene._cache_tilestore,
queue_store=self.gene._queue_tilestore,
count=[self.gene._count_tiles, self.gene._count_tiles_dropped],
proj4_literal=grid["proj4_literal"],
)
else:
return MapnikTileStore(
tilegrid=self.gene._gene.get_grid(config, layer["grid"]),
mapfile=layer["mapfile"],
image_buffer=layer["meta_buffer"] if layer.get("meta") else 0,
data_buffer=layer.get("data_buffer", 128),
output_format=cast(str, layer.get("output_format", "png")),
proj4_literal=grid["proj4_literal"],
)
return None
def gene(self, options, gene, layer):
count_metatiles = None
count_metatiles_dropped = Count()
count_tiles = None
count_tiles_dropped = Count()
if options.role == 'slave' or options.get_hash or options.get_bbox:
gene.layer = gene.layers[layer]
else:
gene.set_layer(layer, options)
if options.get_bbox:
try:
tilecoord = parse_tilecoord(options.get_bbox)
print \
"Tile bounds: [%i,%i,%i,%i]" % \
gene.layer['grid_ref']['obj'].extent(tilecoord)
exit()
except ValueError as e: # pragma: no cover
exit(
"Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r"
% (options.get_bbox, e))
if options.get_hash:
options.role = 'hash'
options.test = 1
sqs_tilestore = None
if options.role in ('master', 'slave'):
# Create SQS queue
sqs_tilestore = SQSTileStore(
gene.get_sqs_queue()) # pragma: no cover
cache_tilestore = None
if options.role in ('local', 'slave'):
cache_tilestore = gene.get_tilesstore(options.cache)
meta = gene.layer['meta']
if options.tiles:
gene.set_store(TilesFileStore(options.tiles))
elif options.role in ('local', 'master'):
# Generate a stream of metatiles
gene.init_tilecoords()
gene.add_geom_filter()
elif options.role == 'slave':
# Get the metatiles from the SQS queue
gene.set_store(sqs_tilestore) # pragma: no cover
elif options.role == 'hash':
try:
z, x, y = (int(v) for v in options.get_hash.split('/'))
if meta:
gene.set_tilecoords(
[TileCoord(z, x, y, gene.layer['meta_size'])])
else:
gene.set_tilecoords([TileCoord(z, x, y)])
except ValueError as e: # pragma: no cover
exit("Tile '%s' is not in the format 'z/x/y'\n%r" %
(options.get_hash, e))
# At this stage, the tilestream contains metatiles that intersect geometry
gene.add_logger()
count_metatiles = gene.counter()
if options.role == 'master': # pragma: no cover
# Put the metatiles into the SQS queue
gene.put(sqs_tilestore)
elif options.role in ('local', 'slave', 'hash'):
if gene.layer['type'] == 'wms':
params = gene.layer['params'].copy()
if 'STYLES' not in params:
params['STYLES'] = ','.join(gene.layer['wmts_style']
for l in gene.layer['layers'])
if gene.layer['generate_salt']:
params['SALT'] = str(random.randint(0, sys.maxint))
for dim in gene.layer['dimensions']:
params[dim['name']] = dim['value']
for dim in gene.options.dimensions:
dim = dim.split('=')
if len(dim) != 2: # pragma: no cover
exit('the DIMENTIONS option should be like this '
'DATE=2013 VERSION=13.')
params[dim[0]] = dim[1]
# Get the metatile image from the WMS server
gene.get(
URLTileStore(
tilelayouts=(WMSTileLayout(
url=gene.layer['url'],
layers=','.join(gene.layer['layers']),
srs=gene.layer['grid_ref']['srs'],
format=gene.layer['mime_type'],
border=gene.layer['meta_buffer'] if meta else 0,
tilegrid=gene.get_grid()['obj'],
params=params,
), ),
headers=gene.layer['headers'],
), "Get tile from WMS")
elif gene.layer['type'] == 'mapnik':
from tilecloud.store.mapnik_ import MapnikTileStore
from tilecloud_chain.mapnik_ import MapnikDropActionTileStore
grid = gene.get_grid()
if gene.layer['output_format'] == 'grid':
gene.get(
MapnikDropActionTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer']
if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
resolution=gene.layer['resolution'],
layers_fields=gene.layer['layers_fields'],
drop_empty_utfgrid=gene.
layer['drop_empty_utfgrid'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
proj4_literal=grid['proj4_literal'],
), "Create Mapnik grid tile")
else:
gene.get(
MapnikTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer']
if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
proj4_literal=grid['proj4_literal'],
), "Create Mapnik tile")
def wrong_content_type_to_error(tile):
if tile is not None and tile.content_type is not None \
and tile.content_type.find("image/") != 0:
if tile.content_type.find(
"application/vnd.ogc.se_xml") == 0:
tile.error = "WMS server error: %s" % (
self._re_rm_xml_tag.sub('', tile.data))
else: # pragma: no cover
tile.error = "%s is not an image format, error: %s" % (
tile.content_type, tile.data)
return tile
gene.imap(wrong_content_type_to_error)
# Handle errors
gene.add_error_filters()
if meta:
if options.role == 'hash':
gene.imap(HashLogger('empty_metatile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_metatile_detection' in gene.layer:
empty_tile = gene.layer['empty_metatile_detection']
gene.imap(
HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_metatiles_dropped,
))
def add_elapsed_togenerate(metatile):
if metatile is not None:
metatile.elapsed_togenerate = metatile.tilecoord.n**2
return True
return False # pragma: no cover
gene.ifilter(add_elapsed_togenerate)
# Split the metatile image into individual tiles
gene.add_metatile_splitter()
gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))
# Handle errors
gene.add_error_filters()
self.count_tiles = gene.counter()
if 'pre_hash_post_process' in gene.layer:
gene.process(gene.layer['pre_hash_post_process'])
if options.role == 'hash':
gene.imap(HashLogger('empty_tile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_tile_detection' in gene.layer:
empty_tile = gene.layer['empty_tile_detection']
gene.imap(
HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
))
gene.process()
if options.role in ('local', 'slave'):
gene.add_error_filters()
gene.ifilter(DropEmpty(gene))
count_tiles = gene.counter(size=True)
if options.time:
def log_size(tile):
sys.stdout.write('size: %i\n' % len(tile.data))
return tile
gene.imap(log_size)
gene.put(cache_tilestore, "Store the tile")
else:
count_tiles = gene.counter(size=True)
gene.add_error_filters()
if options.generated_tiles_file: # pragma: no cover
generated_tiles_file = open(options.generated_tiles_file, 'a')
def do(tile):
generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
return tile
gene.imap(do)
if options.role == 'slave': # pragma: no cover
if meta:
def decr_tile_in_metatile(tile):
tile.metatile.elapsed_togenerate -= 1
if tile.metatile.elapsed_togenerate == 0:
sqs_tilestore.delete_one(tile.metatile)
return True
gene.ifilter(decr_tile_in_metatile)
else:
gene.delete(sqs_tilestore)
if options.time is not None:
class LogTime:
n = 0
t1 = None
def __call__(self, tile):
self.n += 1
if self.n == options.time:
self.t1 = datetime.now()
elif self.n == 2 * options.time:
t2 = datetime.now()
d = (t2 - self.t1) / options.time
sys.stdout.write(
'time: %i\n' %
((d.days * 24 * 3600 + d.seconds) * 1000000 +
d.microseconds))
return tile
gene.imap(LogTime())
gene.consume(options.time * 3)
else:
gene.consume()
if not options.quiet and options.role in ('local', 'slave'):
nb_tiles = count_tiles.nb + count_tiles_dropped.nb
print """The tile generation of layer '%s' is finish
%sNb generated tiles: %i
Nb tiles dropped: %i
Nb tiles stored: %i
Nb error: %i
Total time: %s
Total size: %s
Time per tiles: %i ms
Size per tile: %i o
""" % \
(
gene.layer['name'],
"""Nb generated metatiles: %i
Nb metatiles dropped: %i
""" %
(
count_metatiles.nb, count_metatiles_dropped.nb
) if meta else '',
nb_tiles,
count_tiles_dropped.nb,
count_tiles.nb,
gene.error,
duration_format(gene.duration),
size_format(count_tiles.size),
(gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0,
count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1
)
if cache_tilestore is not None and hasattr(cache_tilestore,
'connection'):
cache_tilestore.connection.close()
if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Layer: %(layer)s
Role: %(role)s
Host: %(host)s
Command: %(cmd)s
%(meta)sNb generated tiles: %(nb_tiles)i
Nb tiles dropped: %(nb_tiles_dropped)i
Total time: %(duration)s [s]
Time per tiles: %(tile_duration)i [ms]""" % {
'role':
options.role,
'layer':
gene.layer['name'],
'host':
socket.getfqdn(),
'cmd':
' '.join([quote(arg) for arg in sys.argv]),
'meta':
"""Nb generated metatiles: %(nb_metatiles)i
Nb metatiles dropped: %(nb_metatiles_dropped)i
""" % {
'nb_metatiles': count_metatiles.nb,
'nb_metatiles_dropped': count_metatiles_dropped.nb,
} if meta else '',
'nb_tiles':
nb_tiles if meta else count_metatiles.nb,
'nb_tiles_dropped':
count_tiles_dropped.nb
if meta else count_metatiles_dropped.nb,
'duration':
duration_format(gene.duration),
'tile_duration': (gene.duration / nb_tiles *
1000).seconds if nb_tiles != 0 else 0,
}, "Tile generation (%(layer)s - %(role)s)" % {
'role': options.role,
'layer': gene.layer['name']
})
def _get_tilestore_for_layer(self, layer) -> Optional[TileStore]:
if layer["type"] == "wms":
params = layer["params"].copy()
if "STYLES" not in params:
params["STYLES"] = ",".join(
layer["wmts_style"] for _ in layer["layers"].split(","))
if layer["generate_salt"]:
params["SALT"] = str(random.randint(0, 999999))
# Get the metatile image from the WMS server
return URLTileStore(
tilelayouts=(WMSTileLayout(
url=layer["url"],
layers=layer["layers"],
srs=layer["grid_ref"]["srs"],
format=layer["mime_type"],
border=layer["meta_buffer"]
if layer.get("meta", False) else 0,
tilegrid=self._gene.get_grid(layer)["obj"],
params=params,
), ),
headers=layer["headers"],
)
elif layer["type"] == "mapnik": # pragma: no cover
try:
from tilecloud.store.mapnik_ import MapnikTileStore # pylint: disable=import-outside-toplevel
from tilecloud_chain.mapnik_ import ( # pylint: disable=import-outside-toplevel
MapnikDropActionTileStore, )
except ImportError:
if os.environ.get("CI", "FALSE") == "FALSE": # pragma nocover
logger.error("Mapnik is not available", exc_info=True)
return None
grid = self._gene.get_grid(layer)
if layer["output_format"] == "grid":
return MapnikDropActionTileStore(
tilegrid=grid["obj"],
mapfile=layer["mapfile"],
image_buffer=layer["meta_buffer"]
if layer.get("meta") else 0,
data_buffer=layer["data_buffer"],
output_format=layer["output_format"],
resolution=layer["resolution"],
layers_fields=layer["layers_fields"],
drop_empty_utfgrid=layer["drop_empty_utfgrid"],
store=self._cache_tilestore,
queue_store=self._queue_tilestore,
count=[self._count_tiles, self._count_tiles_dropped],
proj4_literal=grid["proj4_literal"],
)
else:
return MapnikTileStore(
tilegrid=grid["obj"],
mapfile=layer["mapfile"],
image_buffer=layer["meta_buffer"]
if layer.get("meta") else 0,
data_buffer=layer["data_buffer"],
output_format=layer["output_format"],
proj4_literal=grid["proj4_literal"],
)
return None
def _gene(self, options, gene, layer, dimensions={}):
count_metatiles = None
count_metatiles_dropped = Count()
count_tiles = None
count_tiles_dropped = Count()
count_tiles_stored = None
if options.get_bbox:
try:
tilecoord = parse_tilecoord(options.get_bbox)
print("Tile bounds: [%i,%i,%i,%i]" %
gene.layer['grid_ref']['obj'].extent(tilecoord))
exit()
except ValueError as e: # pragma: no cover
exit(
"Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r"
% (options.get_bbox, e))
if options.get_hash:
options.role = 'hash'
options.test = 1
sqs_tilestore = None
if options.role in ('master', 'slave'):
# Create SQS queue
sqs_tilestore = SQSTileStore(
gene.get_sqs_queue()) # pragma: no cover
cache_tilestore = None
if options.role in ('local', 'slave'):
cache_tilestore = gene.get_tilesstore(options.cache, dimensions)
meta = gene.layer['meta']
if options.tiles:
gene.set_store(TilesFileStore(options.tiles))
elif options.role in ('local', 'master'):
# Generate a stream of metatiles
gene.init_tilecoords()
gene.add_geom_filter()
if options.local_process_number is not None: # pragma: no cover
gene.add_local_process_filter()
elif options.role == 'slave':
# Get the metatiles from the SQS queue
gene.set_store(sqs_tilestore) # pragma: no cover
elif options.role == 'hash':
try:
z, x, y = (int(v) for v in options.get_hash.split('/'))
if meta:
gene.set_tilecoords(
[TileCoord(z, x, y, gene.layer['meta_size'])])
else:
gene.set_tilecoords([TileCoord(z, x, y)])
except ValueError as e: # pragma: no cover
exit("Tile '%s' is not in the format 'z/x/y'\n%r" %
(options.get_hash, e))
# At this stage, the tilestream contains metatiles that intersect geometry
gene.add_logger()
count_metatiles = gene.counter()
if options.role == 'master': # pragma: no cover
# Put the metatiles into the SQS queue
gene.put(sqs_tilestore)
count_tiles = gene.counter()
elif options.role in ('local', 'slave', 'hash'):
if gene.layer['type'] == 'wms':
params = gene.layer['params'].copy()
if 'STYLES' not in params:
params['STYLES'] = ','.join(
gene.layer['wmts_style']
for l in gene.layer['layers'].split(','))
if gene.layer['generate_salt']:
params['SALT'] = str(random.randint(0, 999999))
params.update(dimensions)
# Get the metatile image from the WMS server
gene.get(
URLTileStore(
tilelayouts=(WMSTileLayout(
url=gene.layer['url'],
layers=gene.layer['layers'],
srs=gene.layer['grid_ref']['srs'],
format=gene.layer['mime_type'],
border=gene.layer['meta_buffer'] if meta else 0,
tilegrid=gene.get_grid()['obj'],
params=params,
), ),
headers=gene.layer['headers'],
), "Get tile from WMS")
elif gene.layer['type'] == 'mapnik': # pragma: no cover
from tilecloud.store.mapnik_ import MapnikTileStore
from tilecloud_chain.mapnik_ import MapnikDropActionTileStore
grid = gene.get_grid()
if gene.layer['output_format'] == 'grid':
count_tiles = gene.counter()
gene.get(
MapnikDropActionTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer']
if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
resolution=gene.layer['resolution'],
layers_fields=gene.layer['layers_fields'],
drop_empty_utfgrid=gene.
layer['drop_empty_utfgrid'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
proj4_literal=grid['proj4_literal'],
), "Create Mapnik grid tile")
else:
gene.get(
MapnikTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer']
if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
proj4_literal=grid['proj4_literal'],
), "Create Mapnik tile")
def wrong_content_type_to_error(tile):
if tile is not None and tile.content_type is not None \
and tile.content_type.find("image/") != 0:
if tile.content_type.find(
"application/vnd.ogc.se_xml") == 0:
tile.error = "WMS server error: %s" % (
self._re_rm_xml_tag.sub(
'',
tile.data.decode('utf-8')
if PY3 else tile.data))
else: # pragma: no cover
tile.error = "%s is not an image format, error: %s" % (
tile.content_type, tile.data)
return tile
gene.imap(wrong_content_type_to_error)
# Handle errors
gene.add_error_filters()
if meta:
if options.role == 'hash':
gene.imap(HashLogger('empty_metatile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_metatile_detection' in gene.layer:
empty_tile = gene.layer['empty_metatile_detection']
gene.imap(
HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_metatiles_dropped,
))
def add_elapsed_togenerate(metatile):
if metatile is not None:
metatile.elapsed_togenerate = metatile.tilecoord.n**2
return True
return False # pragma: no cover
gene.ifilter(add_elapsed_togenerate)
# Split the metatile image into individual tiles
gene.add_metatile_splitter()
gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))
# Handle errors
gene.add_error_filters()
if gene.layer['type'] != 'mapnik' or gene.layer[
'output_format'] != 'grid':
count_tiles = gene.counter()
if 'pre_hash_post_process' in gene.layer: # pragma: no cover
gene.process(gene.layer['pre_hash_post_process'])
if options.role == 'hash':
gene.imap(HashLogger('empty_tile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_tile_detection' in gene.layer:
empty_tile = gene.layer['empty_tile_detection']
gene.imap(
HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
))
gene.process()
else: # pragma: no cover
count_tiles = gene.counter()
if options.role in ('local', 'slave'):
gene.add_error_filters()
gene.ifilter(DropEmpty(gene))
count_tiles_stored = gene.counter(size=True)
if options.time:
def log_size(tile):
sys.stdout.write('size: %i\n' % len(tile.data))
return tile
gene.imap(log_size)
gene.put(cache_tilestore, "Store the tile")
gene.add_error_filters()
if options.generated_tiles_file: # pragma: no cover
generated_tiles_file = open(options.generated_tiles_file, 'a')
def do(tile):
generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
return tile
gene.imap(do)
if options.role == 'slave': # pragma: no cover
if meta:
def decr_tile_in_metatile(tile):
tile.metatile.elapsed_togenerate -= 1
if tile.metatile.elapsed_togenerate == 0:
sqs_tilestore.delete_one(tile.metatile)
return True
gene.ifilter(decr_tile_in_metatile)
else:
gene.delete(sqs_tilestore)
message = []
if options.time is not None:
class LogTime:
n = 0
t1 = None
def __call__(self, tile):
self.n += 1
if self.n == options.time:
self.t1 = datetime.now()
elif self.n == 2 * options.time:
t2 = datetime.now()
d = (t2 - self.t1) / options.time
sys.stdout.write(
'time: %i\n' %
((d.days * 24 * 3600 + d.seconds) * 1000000 +
d.microseconds))
return tile
gene.imap(LogTime())
gene.consume(options.time * 3)
else:
gene.consume()
message = [
"The tile generation of layer '{}{}' is finish".format(
gene.layer['name'], "" if len(dimensions) == 0
or gene.layer['type'] != 'wms' else " (%s)" %
", ".join(["=".join(d) for d in dimensions.items()])),
]
if options.role == "master": # pragma: no cover
message.append("Nb of generated jobs: {}".format(
count_tiles.nb))
else:
if meta:
message += [
"Nb generated metatiles: {}".format(
count_metatiles.nb),
"Nb metatiles dropped: {}".format(
count_metatiles_dropped.nb),
]
message += [
"Nb generated tiles: {}".format(count_tiles.nb),
"Nb tiles dropped: {}".format(count_tiles_dropped.nb),
]
if options.role in ('local', 'slave'):
message += [
"Nb tiles stored: {}".format(count_tiles_stored.nb),
"Nb tiles in error: {}".format(gene.error),
"Total time: {}".format(duration_format(
gene.duration)),
]
if count_tiles_stored.nb != 0:
message.append("Total size: {}".format(
size_format(count_tiles_stored.size)))
if count_tiles.nb != 0:
message.append("Time per tile: {:0.0f} ms".format(
(gene.duration / count_tiles.nb * 1000).seconds))
if count_tiles_stored.nb != 0:
message.append("Size per tile: {:0.0f} o".format(
count_tiles_stored.size / count_tiles_stored.nb))
if not options.quiet and options.role in ('local', 'slave'):
print("\n".join(message) + "\n")
if cache_tilestore is not None and hasattr(cache_tilestore,
'connection'):
cache_tilestore.connection.close()
if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
sns_message = [message[0]]
sns_message += [
"Layer: {}".format(gene.layer['name']),
"Role: {}".format(options.role),
"Host: {}".format(socket.getfqdn()),
"Command: {}".format(' '.join([quote(arg)
for arg in sys.argv])),
]
sns_message += message[1:]
connection.publish(
gene.config['sns']['topic'], "\n".join(sns_message),
"Tile generation (%(layer)s - %(role)s)" % {
'role': options.role,
'layer': gene.layer['name']
})
