def process() -> None:
"""Copy the tiles from a cache to an other."""
try:
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_common_options(parser, near=False, time=False, dimensions=True)
parser.add_argument("process",
metavar="PROCESS",
help="The process name to do")
options = parser.parse_args()
gene = TileGeneration(options.config, options, multi_thread=False)
copy = Copy()
if options.layer:
copy.copy(options, gene, options.layer, options.cache,
options.cache, "process")
else:
assert gene.config_file
config = gene.get_config(gene.config_file)
layers_name = (config.config["generation"]["default_layers"]
if "default_layers" in config.config.get(
"generation", {}) else
config.config["layers"].keys())
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache,
"process")
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
sys.exit(1)
def main():
parser = ArgumentParser(
description='Used to calculate the generation cost',
prog=sys.argv[0]
)
add_comon_options(parser, tile_pyramid=False)
parser.add_argument(
'--cost-algo', '--calculate-cost-algorithm', default='area', dest='cost_algo',
choices=('area', 'count'),
help="The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate."
)
options = parser.parse_args()
gene = TileGeneration(
options.config, options,
layer_name=options.layer, base_config={'cost': {}}
)
all_size = 0
tile_size = 0
all_tiles = 0
if (options.layer):
(all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options)
tile_size = gene.layer['cost']['tile_size'] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer in gene.config['generation']['default_layers']:
print("")
print("===== %s =====" % layer)
gene.set_layer(layer, options)
(size, time, price, tiles) = _calculate_cost(gene, options)
tile_size += gene.layer['cost']['tile_size'] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print("")
print("===== GLOBAL =====")
print("Total number of tiles: %i" % all_tiles)
print('Total generation time: %s [d h:mm:ss]' % (duration_format(all_time)))
print('Total generation cost: %0.2f [$]' % all_price)
print("")
print('S3 Storage: %0.2f [$/month]' % (all_size * gene.config['cost']['s3']['storage'] / (1024.0 * 1024 * 1024)))
print('S3 get: %0.2f [$/month]' % (
gene.config['cost']['s3']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
gene.config['cost']['s3']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
)
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
if 'ec2' in gene.config:
print('ESB storage: %0.2f [$/month]' % (
gene.config['cost']['esb']['storage'] * gene.config['cost']['esb_size'])
)
sys.exit(0)
def _fill_legend(
gene: TileGeneration,
cache: tilecloud_chain.configuration.Cache,
server: Optional[tilecloud_chain.configuration.Server],
base_urls: List[str],
) -> None:
assert gene.config_file
config = gene.get_config(gene.config_file)
for layer_name, layer in config.config["layers"].items():
previous_legend: Optional[tilecloud_chain.Legend] = None
previous_resolution = None
if "legend_mime" in layer and "legend_extension" in layer and layer_name not in gene.layer_legends:
gene.layer_legends[layer_name] = []
legends = gene.layer_legends[layer_name]
for zoom, resolution in enumerate(
config.config["grids"][layer["grid"]]["resolutions"]):
path = "/".join([
"1.0.0",
layer_name,
layer["wmts_style"],
f"legend{zoom}.{layer['legend_extension']}",
])
img = _get(path, cache)
if img is not None:
new_legend: tilecloud_chain.Legend = {
"mime_type":
layer["legend_mime"],
"href":
os.path.join(
base_urls[0],
server.get("static_path", "static") +
"/" if server else "", path),
}
legends.append(new_legend)
if previous_legend is not None:
assert previous_resolution is not None
middle_res = exp(
(log(previous_resolution) + log(resolution)) / 2)
previous_legend[ # pylint: disable=unsupported-assignment-operation
"min_resolution"] = middle_res
new_legend["max_resolution"] = middle_res
try:
pil_img = Image.open(BytesIO(img))
new_legend["width"] = pil_img.size[0]
new_legend["height"] = pil_img.size[1]
except Exception: # pragma: nocover
logger.warning(
"Unable to read legend image '%s', with '%s'",
path,
repr(img),
exc_info=True,
)
previous_legend = new_legend
previous_resolution = resolution
def init_tilegeneration(config_file: Optional[str]) -> None:
"""Initialize the tile generation."""
global tilegeneration # pylint: disable=global-statement
if tilegeneration is None:
logger.info("Config file: '%s'", config_file)
log_level = os.environ.get("TILE_SERVER_LOGLEVEL")
tilegeneration = TileGeneration(
config_file,
collections.namedtuple( # type: ignore
"Options",
[
"verbose", "debug", "quiet", "bbox", "zoom", "test",
"near", "time", "geom", "ignore_error"
],
)(
log_level == "verbose", # type: ignore
log_level == "debug",
log_level == "quiet",
None,
None,
None,
None,
None,
True,
False,
),
configure_logging=False,
multi_thread=False,
maxconsecutive_errors=False,
)
def main():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other',
prog=sys.argv[0])
add_comon_options(parser,
near=False,
time=False,
dimensions=True,
cache=False)
parser.add_argument('--process',
dest='process',
metavar="NAME",
help='The process name to do')
parser.add_argument('source', metavar="SOURCE", help='The source cache')
parser.add_argument('dest', metavar="DEST", help='The destination cache')
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if options.layer: # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.source, options.dest,
'copy')
else:
layers = gene.config['generation']['default_layers'] \
if 'default_layers' in gene.config['generation'] \
else gene.config['layers'].keys()
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest,
'copy')
def main():
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_comon_options(parser,
near=False,
time=False,
dimensions=True,
cache=False)
parser.add_argument("--process",
dest="process",
metavar="NAME",
help="The process name to do")
parser.add_argument("source", metavar="SOURCE", help="The source cache")
parser.add_argument("dest", metavar="DEST", help="The destination cache")
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if options.layer: # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.source, options.dest,
"copy")
else:
layers = (gene.config["generation"]["default_layers"]
if "default_layers" in gene.config["generation"] else
gene.config["layers"].keys())
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest,
"copy")
def main() -> None:
"""Copy the tiles from a cache to an other."""
try:
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_common_options(parser,
near=False,
time=False,
dimensions=True,
cache=False)
parser.add_argument("--process",
dest="process",
metavar="NAME",
help="The process name to do")
parser.add_argument("source",
metavar="SOURCE",
help="The source cache")
parser.add_argument("dest",
metavar="DEST",
help="The destination cache")
options = parser.parse_args()
gene = TileGeneration(options.config, options)
assert gene.config_file
config = gene.get_config(gene.config_file)
if options.layer:
copy = Copy()
copy.copy(options, gene, options.layer, options.source,
options.dest, "copy")
else:
layers = (config.config["generation"]["default_layers"]
if "default_layers" in config.config["generation"] else
config.config["layers"].keys())
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest,
"copy")
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
if os.environ.get("TESTS", "false").lower() == "true":
raise
sys.exit(1)
def get_wmts_capabilities(gene: TileGeneration,
cache_name: str,
exit_: bool = False) -> Optional[str]:
"""Get the WMTS capabilities for a configuration file."""
assert gene.config_file
config = gene.get_config(gene.config_file)
cache = config.config["caches"][cache_name]
if _validate_generate_wmts_capabilities(cache, cache_name, exit_):
server = gene.get_main_config().config.get("server")
base_urls = _get_base_urls(cache)
_fill_legend(gene, cache, server, base_urls)
data = pkgutil.get_data("tilecloud_chain",
"wmts_get_capabilities.jinja")
assert data
return cast(
str,
jinja2_template(
data.decode("utf-8"),
layers=config.config["layers"],
layer_legends=gene.layer_legends,
grids=config.config["grids"],
getcapabilities=urljoin( # type: ignore
base_urls[0],
(server.get("wmts_path", "wmts") +
"/1.0.0/WMTSCapabilities.xml" if server is not None else
cache.get("wmtscapabilities_file",
"1.0.0/WMTSCapabilities.xml")),
),
base_urls=base_urls,
base_url_postfix=(server.get("wmts_path", "wmts") +
"/") if server is not None else "",
get_tile_matrix_identifier=get_tile_matrix_identifier,
server=server is not None,
has_metadata="metadata" in config.config,
metadata=config.config.get("metadata"),
has_provider="provider" in config.config,
provider=config.config.get("provider"),
enumerate=enumerate,
ceil=math.ceil,
int=int,
sorted=sorted,
),
)
return None
def get_status(gene: TileGeneration) -> List[str]:
"""Get the tile generation status."""
config = gene.get_main_config()
store = get_queue_store(config, False)
prefix = "redis" if "redis" in config.config else "sqs"
conf = config.config[
"redis"] if "redis" in config.config else config.config["sqs"]
stats_prefix = [prefix, conf["queue"]]
with stats.timer_context(stats_prefix + ["get_stats"]):
status_ = store.get_status()
return [name + ": " + str(value) for name, value in status_.items()]
def test_near(self):
class Opt:
zoom = '3'
test = 196
near = [600000.0, 200000.0]
quiet = False
verbose = False
debug = False
time = None
logging_configuration_file = None
gene = TileGeneration('tilecloud_chain/tests/tilegeneration/test.yaml', Opt(), 'point')
self.assertEqual(gene.geoms[3].bounds, (583840.0, 173360.0, 624800.0, 214320.0))
def process():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other',
prog='./buildout/bin/generate_copy')
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument('process',
metavar="PROCESS",
help='The process name to do')
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if (options.layer): # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.cache, options.cache,
'process')
else:
for layer in gene.config['generation']['default_layers']:
copy = Copy()
copy.copy(options, gene, layer, options.cache, options.cache,
'process')
def process():
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument("process",
metavar="PROCESS",
help="The process name to do")
options = parser.parse_args()
gene = TileGeneration(options.config, options, multi_thread=False)
copy = Copy()
if options.layer: # pragma: no cover
copy.copy(options, gene, options.layer, options.cache, options.cache,
"process")
else:
layers_name = (gene.config["generation"]["default_layers"]
if "default_layers" in gene.config.get(
"generation", {}) else gene.layers.keys())
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache,
"process")
def process():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other',
prog=sys.argv[0])
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument('process',
metavar="PROCESS",
help='The process name to do')
options = parser.parse_args()
gene = TileGeneration(options.config, options)
copy = Copy()
if options.layer: # pragma: no cover
copy.copy(options, gene, options.layer, options.cache, options.cache,
'process')
else:
layers_name = gene.config['generation']['default_layers'] \
if 'default_layers' in gene.config.get('generation', {}) \
else gene.layers.keys()
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache,
'process')
def init_tilegeneration(config_file):
global tilegeneration # pylint: disable=global-statement
if tilegeneration is None:
logger.info("Config file: '%s'", config_file)
log_level = os.environ.get("TILE_SERVER_LOGLEVEL")
tilegeneration = TileGeneration(
config_file,
collections.namedtuple(
"Options",
["verbose", "debug", "quiet", "bbox", "zoom", "test", "near", "time", "geom"],
)(
log_level == "verbose",
log_level == "debug",
log_level == "quiet",
None,
None,
None,
None,
None,
True,
),
configure_logging=False,
multi_thread=False,
)
def main():
parser = ArgumentParser(description='Used to generate the tiles',
prog='./buildout/bin/generate_tiles')
add_comon_options(parser, dimensions=True)
parser.add_argument(
'--get-hash',
metavar="TILE",
help='get the empty tiles hash, use the specified TILE z/x/y')
parser.add_argument(
'--get-bbox',
metavar="TILE",
help=
'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--role',
default='local',
choices=('local', 'master', 'slave'),
help='local/master/slave, master to file the queue and '
'slave to generate the tiles')
parser.add_argument('--daemonize',
default=False,
action="store_true",
help='run as a daemon')
parser.add_argument(
'--tiles',
metavar="FILE",
help=
'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument('--generated-tiles-file',
metavar="FILE",
help='Store the tiles in a file (unrecommended)')
options = parser.parse_args()
if options.daemonize:
daemonize() # pragma: no cover
gene = TileGeneration(options.config, options)
if options.get_hash is None and options.get_bbox is None and \
'authorised_user' in gene.config['generation'] and \
gene.config['generation']['authorised_user'] != getuser():
exit('not authorised, authorised user is: %s.' %
gene.config['generation']['authorised_user'])
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.tiles is not None and options.role not in [
'local', 'master'
]: # pragma: no cover
exit("The --tiles option work only with role local or master")
try:
if (options.layer):
generate = Generate()
generate.gene(options, gene, options.layer)
elif options.get_bbox: # pragma: no cover
exit("With --get-bbox option we needs to specify a layer")
elif options.get_hash: # pragma: no cover
exit("With --get-hash option we needs to specify a layer")
elif options.tiles: # pragma: no cover
exit("With --tiles option we needs to specify a layer")
else:
for layer in gene.config['generation']['default_layers']:
generate = Generate()
generate.gene(options, gene, layer)
finally:
if gene.error_file is not None:
gene.error_file.close()
def test_validate_type(self):
class Opt:
quiet = False
verbose = False
debug = False
test = 0
zoom = None
gene = TileGeneration('tilegeneration/test.yaml', Opt())
obj = {'value': 1}
self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
self.assertEquals(obj['value'], 1)
obj = {'value': 1.0}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': '1 + 1'}
self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
self.assertEquals(obj['value'], 2)
obj = {'value': '1 * 1.5'}
self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
self.assertEquals(obj['value'], 2)
obj = {'value': 'a'}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': {}}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': []}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': 1}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 1.0)
obj = {'value': 1.0}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 1.0)
obj = {'value': '1 + 1'}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 2.0)
obj = {'value': '1 * 1.5'}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 1.5)
obj = {'value': 'a'}
self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
obj = {'value': {}}
self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
obj = {'value': []}
self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
def main():
parser = ArgumentParser(
description='Used to generate the contextual file like the capabilities, the legends, '
'the Apache and MapCache configuration',
prog='./buildout/bin/generate_controller'
)
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument(
'--capabilities', '--generate_wmts_capabilities', default=False, action="store_true",
help='Generate the WMTS Capabilities'
)
parser.add_argument(
'--legends', '--generate_legend_images', default=False, action="store_true", dest='legends',
help='Generate the legend images'
)
parser.add_argument(
'--openlayers', '--generate-openlayers-test-page', default=False,
action="store_true", dest='openlayers',
help='Generate openlayers test page'
)
parser.add_argument(
'--mapcache', '--generate-mapcache-config', default=False, action="store_true", dest='mapcache',
help='Generate MapCache configuration file'
)
parser.add_argument(
'--apache', '--generate-apache-config', default=False, action="store_true", dest='apache',
help='Generate Apache configuration file'
)
parser.add_argument(
'--dump-config', default=False, action="store_true",
help='Dump the used config with default values and exit'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.dump_config:
for layer in gene.config['layers'].keys():
gene.set_layer(layer, options)
validate_calculate_cost(gene)
_validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
gene.validate_mapcache_config()
gene.validate_apache_config()
_validate_generate_openlayers(gene)
for grid in gene.config['grids'].values():
if 'obj' in grid:
del grid['obj']
print yaml.dump(gene.config)
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog='./buildout/bin/generate_amazon'
)
add_comon_options(parser)
parser.add_argument(
'--deploy-config', default=None, dest="deploy_config", metavar="FILE",
help='path to the deploy configuration file'
)
parser.add_argument(
'--status', default=False, action="store_true",
help='display the SQS queue status and exit'
)
parser.add_argument(
'--disable-geodata', default=True, action="store_false", dest="geodata",
help='disable geodata synchronisation'
)
parser.add_argument(
'--disable-code', default=True, action="store_false", dest="deploy_code",
help='disable deploy application code'
)
parser.add_argument(
'--disable-database', default=True, action="store_false", dest="deploy_database",
help='disable deploy database'
)
parser.add_argument(
'--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
help='disable queue filling'
)
parser.add_argument(
'--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
help='disable tile generation'
)
parser.add_argument(
'--host', default=None,
help='The host used to generate tiles'
)
parser.add_argument(
'--shutdown', default=False, action="store_true",
help='Shut done the remote host after the task.'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print "EC2 not configured"
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if options.geodata and 'geodata_folder' in gene.config['ec2']:
print "==== Sync geodata ===="
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options,
'-r', gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code:
print "==== Sync and build code ===="
error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
if error:
exit(1) # pragma: no cover
cmd = ['rsync', '--delete', ]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run_remote(cmd, host, project_dir, gene)
if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
run_remote(
'echo %s > %s' % (
gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']
), host, project_dir, gene
)
run_remote('sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene
)
)
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(
lambda x, y: x + y,
[timedelta(microseconds=int(r)) for r in times],
timedelta()
) / len(times) ** 2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print '==== Time results ===='
print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
print '''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)
if options.shutdown: # pragma: no cover
run_remote('sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue: # pragma: no cover
print "==== Till queue ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master'])
project_dir = gene.config['ec2']['code_folder']
run_remote(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene
)
if options.tiles_gen: # pragma: no cover
print "==== Generate tiles ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'slave'])
arguments.append("--daemonize")
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene)
)
if options.shutdown:
for p in processes:
p.communicate() # wait process end
else:
print 'Tile generation started in background'
if options.shutdown:
run_remote('sudo shutdown 0')
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'],
"""The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
{
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
},
"Tile generation controller")
def main():
stats.init_backends({})
parser = ArgumentParser(description="Used to generate the tiles",
prog=sys.argv[0])
add_comon_options(parser, dimensions=True)
parser.add_argument(
"--get-hash",
metavar="TILE",
help="get the empty tiles hash, use the specified TILE z/x/y")
parser.add_argument(
"--get-bbox",
metavar="TILE",
help=
"get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles",
)
parser.add_argument(
"--role",
default="local",
choices=("local", "master", "slave"),
help=
"local/master/slave, master to file the queue and slave to generate the tiles",
)
parser.add_argument("--local-process-number",
default=None,
help="The number of process that we run in parallel")
parser.add_argument("--detach",
default=False,
action="store_true",
help="run detached from the terminal")
parser.add_argument("--daemon",
default=False,
action="store_true",
help="run continuously as a daemon")
parser.add_argument(
"--tiles",
metavar="FILE",
help=
"Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles",
)
options = parser.parse_args()
if options.detach:
detach() # pragma: no cover
gene = TileGeneration(options.config,
options,
multi_thread=options.get_hash is None)
if (options.get_hash is None and options.get_bbox is None
and "authorised_user" in gene.config["generation"]
and gene.config["generation"]["authorised_user"] != getuser()):
sys.exit("not authorised, authorised user is: {}.".format(
gene.config["generation"]["authorised_user"]))
if options.cache is None:
options.cache = gene.config["generation"]["default_cache"]
if options.tiles is not None and options.role not in [
"local", "master"
]: # pragma: no cover
sys.exit("The --tiles option work only with role local or master")
try:
generate = Generate(options, gene)
if options.role == "slave":
generate.gene()
elif options.layer:
generate.gene(gene.layers[options.layer])
elif options.get_bbox: # pragma: no cover
sys.exit("With --get-bbox option you need to specify a layer")
elif options.get_hash: # pragma: no cover
sys.exit("With --get-hash option you need to specify a layer")
elif options.tiles: # pragma: no cover
sys.exit("With --tiles option you need to specify a layer")
else:
for layer in gene.config["generation"].get("default_layers",
gene.layers.keys()):
generate.gene(gene.layers[layer])
finally:
gene.close()
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog=sys.argv[0]
)
add_comon_options(parser)
parser.add_argument(
'--deploy-config', default=None, dest="deploy_config", metavar="FILE",
help='path to the deploy configuration file'
)
parser.add_argument(
'--status', default=False, action="store_true",
help='display the SQS queue status and exit'
)
parser.add_argument(
'--disable-geodata', default=True, action="store_false", dest="geodata",
help='disable geodata synchronisation'
)
parser.add_argument(
'--disable-code', default=True, action="store_false", dest="deploy_code",
help='disable deploy application code'
)
parser.add_argument(
'--disable-database', default=True, action="store_false", dest="deploy_database",
help='disable deploy database'
)
parser.add_argument(
'--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
help='disable queue filling'
)
parser.add_argument(
'--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
help='disable tile generation'
)
parser.add_argument(
'--host', default=None,
help='The host used to generate tiles'
)
parser.add_argument(
'--shutdown', default=False, action="store_true",
help='Shut done the remote host after the task.'
)
parser.add_argument(
'--wait', default=False, action="store_true",
help='Wait that all the tasks will finish.'
)
parser.add_argument(
'--local', default=False, action="store_true",
help='Run the generation locally'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print("EC2 not configured")
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']:
print("==== Sync geodata ====")
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options,
'-r', gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code and not options.local:
print("==== Sync and build code ====")
error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
if error:
exit(1) # pragma: no cover
cmd = ['rsync', '--delete', ]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run(options, cmd % environ, host, project_dir, gene)
if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
run(
options,
'echo %s > %s' % (
gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']
), host, project_dir, gene
)
run(options, 'sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database and not options.local:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata and not options.local:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = None if options.local else gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
"%sgenerate_tiles %s" % (
_get_path(),
' '.join([str(a) for a in arguments])
), host, project_dir, gene
)
)
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
if PY3:
results = [r.decode('utf-8') for r in results]
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(
lambda x, y: x + y,
[timedelta(microseconds=int(r)) for r in times],
timedelta()
) / len(times) ** 2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print('==== Time results ====')
print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
print('''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))
if options.shutdown: # pragma: no cover
run(options, 'sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue and not options.local: # pragma: no cover
print("==== Till queue ====")
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master', '--quiet'])
project_dir = gene.config['ec2']['code_folder']
run_remote_process(
options,
"%sgenerate_tiles %s" % (
_get_path(),
' '.join([str(a) for a in arguments])
), host, project_dir, gene
)
sleep(5)
attributes = gene.get_sqs_queue().get_attributes()
print(
"\rTiles to generate: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
)
)
if options.tiles_gen: # pragma: no cover
print("==== Generate tiles ====")
if options.wait and not options.local:
print("")
class Status(Thread):
def run(self): # pragma: no cover
while True:
attributes = gene.get_sqs_queue().get_attributes()
print(
"\rTiles to generate/generating: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
)
)
sleep(1)
status_thread = Status()
status_thread.setDaemon(True)
status_thread.start()
arguments = _get_arguments(options)
arguments.extend(['--quiet'])
if not options.local:
arguments.extend(['--role', 'slave'])
project_dir = None if options.local else gene.config['ec2']['code_folder']
threads = []
for i in range(gene.config['ec2']['number_process']):
if options.local:
threads.append(run_local_process(
"%sgenerate_tiles --local-process-number %i %s" % (
_get_path(),
i, ' '.join([str(a) for a in arguments])
)
))
else:
run_remote_process(
"%sgenerate_tiles %s" % (
_get_path(),
' '.join([str(a) for a in arguments])
), host, project_dir, gene
)
print('Tile generation started')
if options.shutdown:
run(options, 'sudo shutdown 0')
if options.wait and options.local:
while len(threads) > 0:
threads = [t for t in threads if t.is_alive()]
sleep(1)
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'],
"""The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
{
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
},
"Tile generation controller"
)
def test_validate_type(self):
class Opt:
quiet = False
verbose = False
debug = False
test = 0
zoom = None
gene = TileGeneration('tilegeneration/test.yaml', Opt())
obj = {'value': 1}
self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
self.assertEquals(obj['value'], 1)
obj = {'value': 1.0}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': '1 + 1'}
self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
self.assertEquals(obj['value'], 2)
obj = {'value': '1 * 1.5'}
self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
self.assertEquals(obj['value'], 2)
obj = {'value': 'a'}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': {}}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': []}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': 1}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 1.0)
obj = {'value': 1.0}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 1.0)
obj = {'value': '1 + 1'}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 2.0)
obj = {'value': '1 * 1.5'}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 1.5)
obj = {'value': 'a'}
self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
obj = {'value': {}}
self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
obj = {'value': []}
self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
def main():
parser = ArgumentParser(
description="Used to calculate the generation cost", prog=sys.argv[0])
add_comon_options(parser, tile_pyramid=False, dimensions=True)
parser.add_argument(
"--cost-algo",
"--calculate-cost-algorithm",
default="area",
dest="cost_algo",
choices=("area", "count"),
help=
"The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate.",
)
options = parser.parse_args()
gene = TileGeneration(options.config,
options,
layer_name=options.layer,
base_config={"cost": {}},
multi_thread=False)
all_size = 0
tile_size = 0
all_tiles = 0
if options.layer:
layer = gene.layers[options.layer]
(all_size, all_time, all_price,
all_tiles) = _calculate_cost(gene, layer, options)
tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer_name in gene.config["generation"]["default_layers"]:
print("")
print("===== {} =====".format(layer_name))
layer = gene.layers[layer_name]
gene.init_layer(layer, options)
(size, time, price, tiles) = _calculate_cost(gene, layer, options)
tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print("")
print("===== GLOBAL =====")
print("Total number of tiles: {}".format(all_tiles))
print("Total generation time: {} [d h:mm:ss]".format(
(duration_format(all_time))))
print("Total generation cost: {0:0.2f} [$]".format(all_price))
print("")
print("S3 Storage: {0:0.2f} [$/month]".format(
all_size * gene.config["cost"]["s3"]["storage"] /
(1024.0 * 1024 * 1024)))
print("S3 get: {0:0.2f} [$/month]".format(
(gene.config["cost"]["s3"]["get"] *
gene.config["cost"]["request_per_layers"] / 10000.0 +
gene.config["cost"]["s3"]["download"] *
gene.config["cost"]["request_per_layers"] * tile_size)))
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] *
# gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] *
# gene.config['cost']['request_per_layers'] * tile_size)
sys.exit(0)
def __init__(self, config_file):
self.filters = {}
self.max_zoom_seed = {}
logger.info("Config file: '%s'" % config_file)
self.tilegeneration = TileGeneration(config_file)
self.expires_hours = self.tilegeneration.config['apache']['expires']
self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \
if 'static_allow_extension' in self.tilegeneration.config['server'] \
else ['jpeg', 'png', 'xml', 'js', 'html', 'css']
self.cache = self.tilegeneration.caches[
self.tilegeneration.config['server']['cache'] if
'cache' in self.tilegeneration.config['server'] else
self.tilegeneration.config['generation']['default_cache']
]
if self.cache['type'] == 's3': # pragma: no cover
s3bucket = S3Connection().bucket(self.cache['bucket'])
def _get(self, path, **kwargs):
global s3bucket
try:
s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path))
responce = s3key.get()
return responce.body, responce.headers['Content-Type']
except:
s3bucket = S3Connection().bucket(self.cache['bucket'])
s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path))
responce = s3key.get()
return responce.body, responce.headers['Content-Type']
else:
folder = self.cache['folder'] or ''
def _get(self, path, **kwargs):
if path.split('.')[-1] not in self.static_allow_extension: # pragma: no cover
return self.error(403, "Extension not allowed", **kwargs), None
p = os.path.join(folder, path)
if not os.path.isfile(p): # pragma: no cover
return self.error(404, path + " not found", **kwargs), None
with open(p, 'rb') as file:
data = file.read()
mime = mimetypes.guess_type(p)
return data, mime[0]
# get capabilities or other static files
self._get = types.MethodType(_get, self)
mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \
'mapcache_base' in self.tilegeneration.config['server'] else \
'http://localhost/'
self.mapcache_baseurl = mapcache_base + self.tilegeneration.config['mapcache']['location'] + '/wmts'
self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \
'mapcache_headers' in self.tilegeneration.config['server'] else {}
geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \
'geoms_redirect' in self.tilegeneration.config['server'] else False
self.layers = self.tilegeneration.config['server']['layers'] if \
'layers' in self.tilegeneration.config['server'] else \
self.tilegeneration.layers.keys()
self.stores = {}
for layer_name in self.layers:
layer = self.tilegeneration.layers[layer_name]
# build geoms redirect
if geoms_redirect:
self.filters[layer_name] = self.tilegeneration.get_geoms_filter(
layer=layer,
grid=layer['grid_ref'],
geoms=self.tilegeneration.get_geoms(
layer,
extent=layer['bbox'] if 'bbox' in layer else layer['grid_ref']['bbox'],
),
)
if 'min_resolution_seed' in layer:
max_zoom_seed = -1
for zoom, resolution in enumerate(layer['grid_ref']['resolutions']):
if resolution > layer['min_resolution_seed']:
max_zoom_seed = zoom
self.max_zoom_seed[layer_name] = max_zoom_seed
else:
self.max_zoom_seed[layer_name] = 999999
# build stores
store_defs = [{
'ref': [layer_name],
'dimensions': {},
}]
for dimension in layer['dimensions']:
new_store_defs = []
for store_def in store_defs:
for value in dimension['values']:
dimensions = {}
dimensions.update(store_def['dimensions'])
dimensions[dimension['name']] = value
new_store_defs.append({
'ref': store_def['ref'] + [value],
'dimensions': dimensions,
})
store_defs = new_store_defs
for store_def in store_defs:
self.stores['/'.join(store_def['ref'])] = \
self.tilegeneration.get_store(self.cache, layer, store_def['dimensions'], read_only=True)
def main():
stats.init_backends({})
parser = ArgumentParser(description='Used to generate the tiles', prog=sys.argv[0])
add_comon_options(parser, dimensions=True)
parser.add_argument(
'--get-hash', metavar="TILE",
help='get the empty tiles hash, use the specified TILE z/x/y'
)
parser.add_argument(
'--get-bbox', metavar="TILE",
help='get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--role', default='local', choices=('local', 'master', 'slave'),
help='local/master/slave, master to file the queue and '
'slave to generate the tiles'
)
parser.add_argument(
"--local-process-number", default=None,
help="The number of process that we run in parallel"
)
parser.add_argument(
'--detach', default=False, action="store_true",
help='run detached from the terminal'
)
parser.add_argument(
'--daemon', default=False, action="store_true",
help='run continuously as a daemon'
)
parser.add_argument(
'--tiles', metavar="FILE",
help='Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
)
options = parser.parse_args()
if options.detach:
detach() # pragma: no cover
gene = TileGeneration(options.config, options)
if options.get_hash is None and options.get_bbox is None and \
'authorised_user' in gene.config['generation'] and \
gene.config['generation']['authorised_user'] != getuser():
exit('not authorised, authorised user is: {}.'.format(gene.config['generation']['authorised_user']))
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.tiles is not None and options.role not in ['local', 'master']: # pragma: no cover
exit("The --tiles option work only with role local or master")
try:
generate = Generate(options, gene)
if options.role == 'slave':
generate.gene()
elif options.layer:
generate.gene(gene.layers[options.layer])
elif options.get_bbox: # pragma: no cover
exit("With --get-bbox option you need to specify a layer")
elif options.get_hash: # pragma: no cover
exit("With --get-hash option you need to specify a layer")
elif options.tiles: # pragma: no cover
exit("With --tiles option you need to specify a layer")
else:
for layer in gene.config['generation'].get('default_layers', gene.layers.keys()):
generate.gene(gene.layers[layer])
finally:
gene.close()
def test_validate_type(self):
class Opt:
quiet = False
verbose = False
debug = False
test = 0
zoom = None
gene = TileGeneration("tilegeneration/test.yaml", Opt())
obj = {"value": 1}
self.assertEqual(gene.validate(obj, "object", "value", int), False)
self.assertEqual(obj["value"], 1)
obj = {"value": 1.0}
self.assertEqual(gene.validate(obj, "object", "value", int), True)
obj = {"value": "1 + 1"}
self.assertEqual(gene.validate(obj, "object", "value", int), False)
self.assertEqual(obj["value"], 2)
obj = {"value": "1 * 1.5"}
self.assertEqual(gene.validate(obj, "object", "value", int), False)
self.assertEqual(obj["value"], 2)
obj = {"value": "a"}
self.assertEqual(gene.validate(obj, "object", "value", int), True)
obj = {"value": {}}
self.assertEqual(gene.validate(obj, "object", "value", int), True)
obj = {"value": []}
self.assertEqual(gene.validate(obj, "object", "value", int), True)
obj = {"value": 1}
self.assertEqual(gene.validate(obj, "object", "value", float), False)
self.assertEqual(obj["value"], 1.0)
obj = {"value": 1.0}
self.assertEqual(gene.validate(obj, "object", "value", float), False)
self.assertEqual(obj["value"], 1.0)
obj = {"value": "1 + 1"}
self.assertEqual(gene.validate(obj, "object", "value", float), False)
self.assertEqual(obj["value"], 2.0)
obj = {"value": "1 * 1.5"}
self.assertEqual(gene.validate(obj, "object", "value", float), False)
self.assertEqual(obj["value"], 1.5)
obj = {"value": "a"}
self.assertEqual(gene.validate(obj, "object", "value", float), True)
obj = {"value": {}}
self.assertEqual(gene.validate(obj, "object", "value", float), True)
obj = {"value": []}
self.assertEqual(gene.validate(obj, "object", "value", float), True)
def __init__(self, config_file):
self.filters = {}
self.max_zoom_seed = {}
logger.info("Config file: '{}'".format(config_file))
self.tilegeneration = TileGeneration(config_file)
self.expires_hours = self.tilegeneration.config['apache']['expires']
self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \
if 'static_allow_extension' in self.tilegeneration.config['server'] \
else ['jpeg', 'png', 'xml', 'js', 'html', 'css']
self.cache = self.tilegeneration.caches[
self.tilegeneration.config['server']['cache'] if 'cache' in
self.tilegeneration.config['server'] else self.tilegeneration.
config['generation']['default_cache']]
if self.cache['type'] == 's3': # pragma: no cover
client = tilecloud.store.s3.get_client(self.cache.get('host'))
bucket = self.cache['bucket']
def _get(self, path, **kwargs):
key_name = os.path.join('{folder}'.format(**self.cache), path)
try:
response = client.get_object(Bucket=bucket, Key=key_name)
return response['Body'].read(), response.get('ContentType')
except Exception:
client = tilecloud.store.s3.get_client(
self.cache.get('host'))
response = client.get_object(Bucket=bucket, Key=key_name)
return response['Body'].read(), response.get('ContentType')
else:
folder = self.cache['folder'] or ''
def _get(self, path, **kwargs):
if path.split(
'.'
)[-1] not in self.static_allow_extension: # pragma: no cover
return self.error(403, "Extension not allowed",
**kwargs), None
p = os.path.join(folder, path)
if not os.path.isfile(p): # pragma: no cover
return self.error(404, path + " not found", **kwargs), None
with open(p, 'rb') as file:
data = file.read()
mime = mimetypes.guess_type(p)
return data, mime[0]
# get capabilities or other static files
self._get = types.MethodType(_get, self)
mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \
'mapcache_base' in self.tilegeneration.config['server'] else \
'http://localhost/'
self.mapcache_baseurl = mapcache_base + self.tilegeneration.config[
'mapcache']['location'] + '/wmts'
self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \
'mapcache_headers' in self.tilegeneration.config['server'] else {}
geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \
'geoms_redirect' in self.tilegeneration.config['server'] else False
self.layers = self.tilegeneration.config['server']['layers'] if \
'layers' in self.tilegeneration.config['server'] else \
self.tilegeneration.layers.keys()
self.stores = {}
for layer_name in self.layers:
layer = self.tilegeneration.layers[layer_name]
# build geoms redirect
if geoms_redirect:
self.filters[
layer_name] = self.tilegeneration.get_geoms_filter(
layer=layer,
grid=layer['grid_ref'],
geoms=self.tilegeneration.get_geoms(
layer,
extent=layer['bbox']
if 'bbox' in layer else layer['grid_ref']['bbox'],
),
)
if 'min_resolution_seed' in layer:
max_zoom_seed = -1
for zoom, resolution in enumerate(
layer['grid_ref']['resolutions']):
if resolution > layer['min_resolution_seed']:
max_zoom_seed = zoom
self.max_zoom_seed[layer_name] = max_zoom_seed
else:
self.max_zoom_seed[layer_name] = 999999
# build stores
store_defs = [{
'ref': [layer_name],
'dimensions': {},
}]
for dimension in layer['dimensions']:
new_store_defs = []
for store_def in store_defs:
for value in dimension['values']:
dimensions = {}
dimensions.update(store_def['dimensions'])
dimensions[dimension['name']] = value
new_store_defs.append({
'ref': store_def['ref'] + [value],
'dimensions': dimensions,
})
store_defs = new_store_defs
for store_def in store_defs:
self.stores['/'.join(store_def['ref'])] = \
self.tilegeneration.get_store(self.cache, layer, read_only=True)
def _calculate_cost(gene: TileGeneration, layer_name: str,
options: Namespace) -> Tuple[float, timedelta, float, int]:
nb_metatiles = {}
nb_tiles = {}
config = gene.get_config(options.config)
layer = config.config["layers"][layer_name]
meta = layer["meta"]
if options.cost_algo == "area":
tile_size = config.config["grids"][layer["grid"]]["tile_size"]
for zoom, resolution in enumerate(
config.config["grids"][layer["grid"]]["resolutions"]):
if "min_resolution_seed" in layer and resolution < layer[
"min_resolution_seed"]:
continue
print(f"Calculate zoom {zoom}.")
px_buffer = layer["px_buffer"] + layer["meta_buffer"] if meta else 0
m_buffer = px_buffer * resolution
if meta:
size = tile_size * layer["meta_size"] * resolution
meta_buffer = size * 0.7 + m_buffer
meta_geom = gene.get_geoms(config, layer_name)[zoom].buffer(
meta_buffer, 1)
nb_metatiles[zoom] = int(round(meta_geom.area / size**2))
size = tile_size * resolution
tile_buffer = size * 0.7 + m_buffer
geom = gene.get_geoms(config,
layer_name)[zoom].buffer(tile_buffer, 1)
nb_tiles[zoom] = int(round(geom.area / size**2))
elif options.cost_algo == "count":
gene.init_tilecoords(config, layer_name)
gene.add_geom_filter()
if meta:
def count_metatile(tile: Tile) -> Tile:
if tile:
if tile.tilecoord.z in nb_metatiles:
nb_metatiles[tile.tilecoord.z] += 1
else:
nb_metatiles[tile.tilecoord.z] = 1
return tile
gene.imap(count_metatile)
class MetaTileSplitter(TileStore):
"""Convert the metatile flow to tile flow."""
@staticmethod
def get(tiles: Iterable[Tile]) -> Iterator[Tile]:
for metatile in tiles:
for tilecoord in metatile.tilecoord:
yield Tile(tilecoord)
gene.add_metatile_splitter(MetaTileSplitter())
# Only keep tiles that intersect geometry
gene.add_geom_filter()
def count_tile(tile: Tile) -> Tile:
if tile:
if tile.tilecoord.z in nb_tiles:
nb_tiles[tile.tilecoord.z] += 1
else:
print(f"Calculate zoom {tile.tilecoord.z}.")
nb_tiles[tile.tilecoord.z] = 1
return tile
gene.imap(count_tile)
run = Run(gene, gene.functions_metatiles)
assert gene.tilestream
for tile in gene.tilestream:
tile.metadata["layer"] = layer_name
run(tile)
times = {}
print()
for z, nb_metatile in nb_metatiles.items():
print(f"{nb_metatile} meta tiles in zoom {z}.")
times[z] = layer["cost"]["metatile_generation_time"] * nb_metatile
price: float = 0
all_size: float = 0
all_time: float = 0
all_tiles = 0
for z, nb_tile in nb_tiles.items():
print()
print(f"{nb_tile} tiles in zoom {z}.")
all_tiles += nb_tile
if meta:
time = times[z] + layer["cost"]["tile_generation_time"] * nb_tile
else:
time = layer["cost"]["tileonly_generation_time"] * nb_tile
size = layer["cost"]["tile_size"] * nb_tile
all_size += size
all_time += time
td = timedelta(milliseconds=time)
print(f"Time to generate: {duration_format(td)} [d h:mm:ss]")
c = gene.get_main_config(
).config["cost"]["s3"]["put"] * nb_tile / 1000.0
price += c
print(f"S3 PUT: {c:0.2f} [$]")
if "sqs" in gene.get_main_config().config:
if meta:
nb_sqs = nb_metatiles[z] * 3
else:
nb_sqs = nb_tile * 3
c = nb_sqs * gene.get_main_config(
).config["cost"]["sqs"]["request"] / 1000000.0
price += c
print(f"SQS usage: {c:0.2f} [$]")
print()
td = timedelta(milliseconds=all_time)
print(f"Number of tiles: {all_tiles}")
print(f"Generation time: {duration_format(td)} [d h:mm:ss]")
print(f"Generation cost: {price:0.2f} [$]")
return all_size, td, price, all_tiles
def __init__(self, config_file):
self.filters = {}
self.max_zoom_seed = {}
logger.info("Config file: '%s'" % config_file)
self.tilegeneration = TileGeneration(config_file)
if not self.tilegeneration.validate_apache_config(): # pragma: no cover
raise "Apache configuration error"
self.expires_hours = self.tilegeneration.config['apache']['expires']
self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \
if 'static_allow_extension' in self.tilegeneration.config['server'] \
else ['jpeg', 'png', 'xml', 'js', 'html', 'css']
self.cache = self.tilegeneration.caches[
self.tilegeneration.config['server']['cache'] if
'cache' in self.tilegeneration.config['server'] else
self.tilegeneration.config['generation']['default_cache']
]
if self.cache['type'] == 's3': # pragma: no cover
s3bucket = S3Connection().bucket(self.cache['bucket'])
def _get(self, path, **kwargs):
global s3bucket
try:
s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path))
responce = s3key.get()
return responce.body, responce.headers['Content-Type']
except:
s3bucket = S3Connection().bucket(self.cache['bucket'])
s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path))
responce = s3key.get()
return responce.body, responce.headers['Content-Type']
else:
folder = self.cache['folder'] or ''
def _get(self, path, **kwargs):
if path.split('.')[-1] not in self.static_allow_extension: # pragma: no cover
return self.error(403, "Extension not allowed", **kwargs), None
p = os.path.join(folder, path)
if not os.path.isfile(p): # pragma: no cover
return self.error(404, path + " not found", **kwargs), None
with open(p, 'rb') as file:
data = file.read()
mime = mimetypes.guess_type(p)
return data, mime[0]
# get capabilities or other static files
self._get = types.MethodType(_get, self)
if not self.tilegeneration.validate_mapcache_config(): # pragma: no cover
raise "Mapcache configuration error"
mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \
'mapcache_base' in self.tilegeneration.config['server'] else \
'http://localhost/'
self.mapcache_baseurl = mapcache_base + self.tilegeneration.config['mapcache']['location'] + '/wmts'
self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \
'mapcache_headers' in self.tilegeneration.config['server'] else {}
geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \
'geoms_redirect' in self.tilegeneration.config['server'] else False
self.layers = self.tilegeneration.config['server']['layers'] if \
'layers' in self.tilegeneration.config['server'] else \
self.tilegeneration.layers.keys()
self.stores = {}
for layer_name in self.layers:
layer = self.tilegeneration.layers[layer_name]
# build geoms redirect
if geoms_redirect:
self.filters[layer_name] = self.tilegeneration.get_geoms_filter(
layer=layer,
grid=layer['grid_ref'],
geoms=self.tilegeneration.get_geoms(
layer,
extent=layer['bbox'] if 'bbox' in layer else layer['grid_ref']['bbox'],
),
)
if 'min_resolution_seed' in layer:
max_zoom_seed = -1
for zoom, resolution in enumerate(layer['grid_ref']['resolutions']):
if resolution > layer['min_resolution_seed']:
max_zoom_seed = zoom
self.max_zoom_seed[layer_name] = max_zoom_seed
else:
self.max_zoom_seed[layer_name] = sys.maxint
# build stores
store_defs = [{
'ref': [layer_name],
'dimensions': [],
}]
for dimension in layer['dimensions']:
new_store_defs = []
for store_def in store_defs:
for value in dimension['values']:
new_store_defs.append({
'ref': store_def['ref'] + [value],
'dimensions': store_def['dimensions'] + [(dimension['name'], value)],
})
store_defs = new_store_defs
for store_def in store_defs:
self.stores['/'.join(store_def['ref'])] = \
self.tilegeneration.get_store(self.cache, layer, store_def['dimensions'], read_only=True)
def main():
parser = ArgumentParser(description='Used to generate the tiles',
prog=sys.argv[0])
add_comon_options(parser, dimensions=True)
parser.add_argument(
'--get-hash',
metavar="TILE",
help='get the empty tiles hash, use the specified TILE z/x/y')
parser.add_argument(
'--get-bbox',
metavar="TILE",
help=
'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--role',
default='local',
choices=('local', 'master', 'slave'),
help='local/master/slave, master to file the queue and '
'slave to generate the tiles')
parser.add_argument("--local-process-number",
default=None,
help="The number of process that we run in parallel")
parser.add_argument('--detach',
default=False,
action="store_true",
help='run detached from the terminal')
parser.add_argument('--daemon',
default=False,
action="store_true",
help='run continuously as a daemon')
parser.add_argument(
'--tiles',
metavar="FILE",
help=
'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument('--generated-tiles-file',
metavar="FILE",
help='Store the tiles in a file (unrecommended)')
options = parser.parse_args()
if options.detach:
detach() # pragma: no cover
gene = TileGeneration(options.config, options)
if options.get_hash is None and options.get_bbox is None and \
'authorised_user' in gene.config['generation'] and \
gene.config['generation']['authorised_user'] != getuser():
exit('not authorised, authorised user is: {}.'.format(
gene.config['generation']['authorised_user']))
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.tiles is not None and options.role not in [
'local', 'master'
]: # pragma: no cover
exit("The --tiles option work only with role local or master")
try:
if options.role == 'slave':
generate = Generate()
generate.gene(options, gene)
elif (options.layer):
generate = Generate()
generate.gene(options, gene, options.layer)
elif options.get_bbox: # pragma: no cover
exit("With --get-bbox option we needs to specify a layer")
elif options.get_hash: # pragma: no cover
exit("With --get-hash option we needs to specify a layer")
elif options.tiles: # pragma: no cover
exit("With --tiles option we needs to specify a layer")
else:
for layer in gene.config['generation'].get('default_layers',
gene.layers.keys()):
generate = Generate()
generate.gene(options, gene, layer)
finally:
if gene.error_file is not None:
gene.error_file.close()
class Server:
def __init__(self, config_file):
self.filters = {}
self.max_zoom_seed = {}
logger.info("Config file: '%s'" % config_file)
self.tilegeneration = TileGeneration(config_file)
self.expires_hours = self.tilegeneration.config['apache']['expires']
self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \
if 'static_allow_extension' in self.tilegeneration.config['server'] \
else ['jpeg', 'png', 'xml', 'js', 'html', 'css']
self.cache = self.tilegeneration.caches[
self.tilegeneration.config['server']['cache'] if
'cache' in self.tilegeneration.config['server'] else
self.tilegeneration.config['generation']['default_cache']
]
if self.cache['type'] == 's3': # pragma: no cover
s3bucket = S3Connection().bucket(self.cache['bucket'])
def _get(self, path, **kwargs):
global s3bucket
try:
s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path))
responce = s3key.get()
return responce.body, responce.headers['Content-Type']
except:
s3bucket = S3Connection().bucket(self.cache['bucket'])
s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path))
responce = s3key.get()
return responce.body, responce.headers['Content-Type']
else:
folder = self.cache['folder'] or ''
def _get(self, path, **kwargs):
if path.split('.')[-1] not in self.static_allow_extension: # pragma: no cover
return self.error(403, "Extension not allowed", **kwargs), None
p = os.path.join(folder, path)
if not os.path.isfile(p): # pragma: no cover
return self.error(404, path + " not found", **kwargs), None
with open(p, 'rb') as file:
data = file.read()
mime = mimetypes.guess_type(p)
return data, mime[0]
# get capabilities or other static files
self._get = types.MethodType(_get, self)
mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \
'mapcache_base' in self.tilegeneration.config['server'] else \
'http://localhost/'
self.mapcache_baseurl = mapcache_base + self.tilegeneration.config['mapcache']['location'] + '/wmts'
self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \
'mapcache_headers' in self.tilegeneration.config['server'] else {}
geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \
'geoms_redirect' in self.tilegeneration.config['server'] else False
self.layers = self.tilegeneration.config['server']['layers'] if \
'layers' in self.tilegeneration.config['server'] else \
self.tilegeneration.layers.keys()
self.stores = {}
for layer_name in self.layers:
layer = self.tilegeneration.layers[layer_name]
# build geoms redirect
if geoms_redirect:
self.filters[layer_name] = self.tilegeneration.get_geoms_filter(
layer=layer,
grid=layer['grid_ref'],
geoms=self.tilegeneration.get_geoms(
layer,
extent=layer['bbox'] if 'bbox' in layer else layer['grid_ref']['bbox'],
),
)
if 'min_resolution_seed' in layer:
max_zoom_seed = -1
for zoom, resolution in enumerate(layer['grid_ref']['resolutions']):
if resolution > layer['min_resolution_seed']:
max_zoom_seed = zoom
self.max_zoom_seed[layer_name] = max_zoom_seed
else:
self.max_zoom_seed[layer_name] = 999999
# build stores
store_defs = [{
'ref': [layer_name],
'dimensions': {},
}]
for dimension in layer['dimensions']:
new_store_defs = []
for store_def in store_defs:
for value in dimension['values']:
dimensions = {}
dimensions.update(store_def['dimensions'])
dimensions[dimension['name']] = value
new_store_defs.append({
'ref': store_def['ref'] + [value],
'dimensions': dimensions,
})
store_defs = new_store_defs
for store_def in store_defs:
self.stores['/'.join(store_def['ref'])] = \
self.tilegeneration.get_store(self.cache, layer, store_def['dimensions'], read_only=True)
def __call__(self, environ, start_response):
params = {}
for key, value in parse_qs(environ['QUERY_STRING'], True).items():
params[key.upper()] = value[0]
path = None if len(params) > 0 else environ['PATH_INFO'][1:].split('/')
return self.serve(path, params, start_response=start_response)
def serve(self, path, params, **kwargs):
dimensions = []
if path is not None:
if len(path) >= 1 and path[0] == 'static':
body, mime = self._get('/'.join(path[1:]), **kwargs)
if mime is not None:
return self.responce(body, {
'Content-Type': mime,
'Expires': (
datetime.datetime.utcnow() +
datetime.timedelta(hours=self.expires_hours)
).isoformat(),
'Cache-Control': "max-age=%i" % (3600 * self.expires_hours),
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET',
}, **kwargs)
else: # pragma: no cover
return body
elif len(path) >= 1 and path[0] != 'wmts': # pragma: no cover
return self.error(
404,
"Type '%s' don't exists, allows values: 'wmts' or 'static'" % path[0],
**kwargs
)
path = path[1:] # remove type
if len(path) == 2 and path[0] == '1.0.0' and path[1].lower() == 'wmtscapabilities.xml':
params['SERVICE'] = 'WMTS'
params['VERSION'] = '1.0.0'
params['REQUEST'] = 'GetCapabilities'
elif len(path) < 7:
return self.error(400, "Not enough path", **kwargs)
else:
params['SERVICE'] = 'WMTS'
params['VERSION'] = path[0]
params['LAYER'] = path[1]
params['STYLE'] = path[2]
if params['LAYER'] in self.layers:
layer = self.tilegeneration.layers[params['LAYER']]
else:
return self.error(400, "Wrong Layer '%s'" % params['LAYER'], **kwargs)
index = 3
dimensions = path[index:index + len(layer['dimensions'])]
for dimension in layer['dimensions']:
params[dimension['name'].upper()] = path[index]
index += 1
last = path[-1].split('.')
if len(path) < index + 4: # pragma: no cover
return self.error(400, "Not enough path", **kwargs)
params['TILEMATRIXSET'] = path[index]
params['TILEMATRIX'] = path[index + 1]
params['TILEROW'] = path[index + 2]
if len(path) == index + 4:
params['REQUEST'] = 'GetTile'
params['TILECOL'] = last[0]
if last[1] != layer['extension']: # pragma: no cover
return self.error(400, "Wrong extention '%s'" % last[1], **kwargs)
elif len(path) == index + 6:
params['REQUEST'] = 'GetFeatureInfo'
params['TILECOL'] = path[index + 3]
params['I'] = path[index + 4]
params['J'] = last[0]
params['INFO_FORMAT'] = layer.get('info_formats', ['application/vnd.ogc.gml'])[0]
else: # pragma: no cover
return self.error(400, "Wrong path length", **kwargs)
params['FORMAT'] = layer['mime_type']
else:
if \
'SERVICE' not in params or \
'REQUEST' not in params or \
'VERSION' not in params:
return self.error(400, "Not all required parameters are present", **kwargs)
if params['SERVICE'] != 'WMTS':
return self.error(400, "Wrong Service '%s'" % params['SERVICE'], **kwargs)
if params['VERSION'] != '1.0.0':
return self.error(400, "Wrong Version '%s'" % params['VERSION'], **kwargs)
if params['REQUEST'] == 'GetCapabilities':
wmtscapabilities_file = self.cache['wmtscapabilities_file']
body, mime = self._get(wmtscapabilities_file, **kwargs)
if mime is not None:
return self.responce(body, headers={
'Content-Type': "application/xml",
'Expires': (
datetime.datetime.utcnow() +
datetime.timedelta(hours=self.expires_hours)
).isoformat(),
'Cache-Control': "max-age=%i" % (3600 * self.expires_hours),
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET',
}, **kwargs)
else: # pragma: no cover
return body
if \
'FORMAT' not in params or \
'LAYER' not in params or \
'TILEMATRIXSET' not in params or \
'TILEMATRIX' not in params or \
'TILEROW' not in params or \
'TILECOL' not in params: # pragma: no cover
return self.error(400, "Not all required parameters are present", **kwargs)
if path is None:
if params['LAYER'] in self.layers:
layer = self.tilegeneration.layers[params['LAYER']]
else:
return self.error(400, "Wrong Layer '%s'" % params['LAYER'], **kwargs)
for dimension in layer['dimensions']:
dimensions.append(
params[dimension['name'].upper()]
if dimension['name'].lower() in params
else dimension['default']
)
if params['STYLE'] != layer['wmts_style']:
return self.error(400, "Wrong Style '%s'" % params['STYLE'], **kwargs)
if params['TILEMATRIXSET'] != layer['grid']:
return self.error(400, "Wrong TileMatrixSet '%s'" % params['TILEMATRIXSET'], **kwargs)
tile = Tile(TileCoord(
# TODO fix for matrix_identifier = resolution
int(params['TILEMATRIX']),
int(params['TILECOL']),
int(params['TILEROW']),
))
if params['REQUEST'] == 'GetFeatureInfo':
if \
'I' not in params or \
'J' not in params or \
'INFO_FORMAT' not in params: # pragma: no cover
return self.error(400, "Not all required parameters are present", **kwargs)
if 'query_layers' in layer:
return self.forward(
layer['url'] + '?' + urlencode({
'SERVICE': 'WMS',
'VERSION': '1.1.1',
'REQUEST': 'GetFeatureInfo',
'LAYERS': layer['layers'],
'QUERY_LAYERS': layer['query_layers'],
'STYLES': params['STYLE'],
'FORMAT': params['FORMAT'],
'INFO_FORMAT': params['INFO_FORMAT'],
'WIDTH': layer['grid_ref']['tile_size'],
'HEIGHT': layer['grid_ref']['tile_size'],
'SRS': layer['grid_ref']['srs'],
'BBOX': layer['grid_ref']['obj'].extent(tile.tilecoord),
'X': params['I'],
'Y': params['J'],
}), no_cache=True, **kwargs
)
else: # pragma: no cover
return self.error(400, "Layer '%s' not queryable" % layer['name'], **kwargs)
if params['REQUEST'] != 'GetTile':
return self.error(400, "Wrong Request '%s'" % params['REQUEST'], **kwargs)
if params['FORMAT'] != layer['mime_type']:
return self.error(400, "Wrong Format '%s'" % params['FORMAT'], **kwargs)
if tile.tilecoord.z > self.max_zoom_seed[layer['name']]: # pragma: no cover
return self.forward(
self.mapcache_baseurl + '?' + urlencode(params),
headers=self.mapcache_header,
**kwargs
)
if layer['name'] in self.filters:
layer_filter = self.filters[layer['name']]
meta_size = layer['meta_size']
meta_tilecoord = TileCoord(
# TODO fix for matrix_identifier = resolution
tile.tilecoord.z,
tile.tilecoord.x / meta_size * meta_size,
tile.tilecoord.y / meta_size * meta_size,
meta_size,
) if meta_size != 1 else tile.tilecoord
if not layer_filter.filter_tilecoord(meta_tilecoord): # pragma: no cover
return self.forward(
self.mapcache_baseurl + '?' + urlencode(params),
headers=self.mapcache_header,
**kwargs
)
store_ref = '/'.join([params['LAYER']] + dimensions)
if store_ref in self.stores: # pragma: no cover
store = self.stores[store_ref]
else: # pragma: no cover
return self.error(
400,
"No store found for layer '%s' and dimensions %s" % (
layer['name'], ', '.join(dimensions)
),
**kwargs
)
tile = store.get_one(tile)
if tile:
return self.responce(tile.data, headers={
'Content-Type': tile.content_type,
'Expires': (
datetime.datetime.utcnow() +
datetime.timedelta(hours=self.expires_hours)
).isoformat(),
'Cache-Control': "max-age=%i" % (3600 * self.expires_hours),
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET',
}, **kwargs)
else:
return self.error(204, **kwargs)
def forward(self, url, headers={}, no_cache=False, **kwargs):
if no_cache:
headers['Cache-Control'] = 'no-cache'
headers['Pragma'] = 'no-cache'
responce = requests.get(url, headers=headers)
if responce.status_code == 200:
responce_headers = responce.headers.copy()
if no_cache:
responce_headers['Cache-Control'] = 'no-cache, no-store'
responce_headers['Pragma'] = 'no-cache'
else: # pragma: no cover
responce_headers['Expires'] = (
datetime.datetime.utcnow() +
datetime.timedelta(hours=self.expires_hours)
).isoformat()
responce_headers['Cache-Control'] = "max-age=%i" % (3600 * self.expires_hours)
responce_headers['Access-Control-Allow-Origin'] = '*'
responce_headers['Access-Control-Allow-Methods'] = 'GET'
responce_headers
return self.responce(responce.content, headers=responce_headers, **kwargs)
else: # pragma: no cover
message = "The URL '%s' return '%i %s', content:\n%s" % (
url, responce.status_code, responce.reason, responce.text,
)
logger.warning(message)
return self.error(502, message=message, **kwargs)
HTTP_MESSAGES = {
204: '204 No Content',
400: '400 Bad Request',
403: '403 Forbidden',
404: '404 Not Found',
502: '502 Bad Gateway',
}
def error(self, code, message='', start_response=None):
start_response(self.HTTP_MESSAGES[code], [])
return [message]
def responce(self, data, headers={}, start_response=None):
headers['Content-Length'] = str(len(data))
start_response('200 OK', headers.items())
return [data]
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog=sys.argv[0])
add_comon_options(parser)
parser.add_argument('--deploy-config',
default=None,
dest="deploy_config",
metavar="FILE",
help='path to the deploy configuration file')
parser.add_argument('--status',
default=False,
action="store_true",
help='display the SQS queue status and exit')
parser.add_argument('--disable-geodata',
default=True,
action="store_false",
dest="geodata",
help='disable geodata synchronisation')
parser.add_argument('--disable-code',
default=True,
action="store_false",
dest="deploy_code",
help='disable deploy application code')
parser.add_argument('--disable-database',
default=True,
action="store_false",
dest="deploy_database",
help='disable deploy database')
parser.add_argument('--disable-fillqueue',
default=True,
action="store_false",
dest="fill_queue",
help='disable queue filling')
parser.add_argument('--disable-tilesgen',
default=True,
action="store_false",
dest="tiles_gen",
help='disable tile generation')
parser.add_argument('--host',
default=None,
help='The host used to generate tiles')
parser.add_argument('--shutdown',
default=False,
action="store_true",
help='Shut done the remote host after the task.')
parser.add_argument('--wait',
default=False,
action="store_true",
help='Wait that all the tasks will finish.')
parser.add_argument('--local',
default=False,
action="store_true",
help='Run the generation locally')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print("EC2 not configured")
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if not options.local and options.geodata and 'geodata_folder' in gene.config[
'ec2']: # pragma: no cover
print("==== Sync geodata ====")
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code and not options.local:
print("==== Sync and build code ====")
cmd = [
'rsync',
'--delete',
]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run(options, cmd % environ, host, project_dir, gene)
if 'apache_content' in gene.config[
'ec2'] and 'apache_config' in gene.config['ec2']:
run(
options,
'echo %s > %s' % (gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']), host,
project_dir, gene)
run(options, 'sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database and not options.local:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata and not options.local:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = None if options.local else gene.config['ec2'][
'code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
"%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene))
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
if PY3:
results = [r.decode('utf-8') for r in results]
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(lambda x, y: x + y,
[timedelta(microseconds=int(r))
for r in times], timedelta()) / len(times)**2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print('==== Time results ====')
print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
print('''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))
if options.shutdown: # pragma: no cover
run(options, 'sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue and not options.local: # pragma: no cover
print("==== Till queue ====")
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master', '--quiet'])
project_dir = gene.config['ec2']['code_folder']
run_remote_process(
options, "%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene)
sleep(5)
attributes = gene.get_sqs_queue().get_attributes()
print("\rTiles to generate: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
))
if options.tiles_gen: # pragma: no cover
print("==== Generate tiles ====")
if options.wait and not options.local:
print("")
class Status(Thread):
def run(self): # pragma: no cover
while True:
attributes = gene.get_sqs_queue().get_attributes()
print("\rTiles to generate/generating: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
))
sleep(1)
status_thread = Status()
status_thread.setDaemon(True)
status_thread.start()
arguments = _get_arguments(options)
arguments.extend(['--quiet'])
if not options.local:
arguments.extend(['--role', 'slave'])
project_dir = None if options.local else gene.config['ec2'][
'code_folder']
threads = []
for i in range(gene.config['ec2']['number_process']):
if options.local:
threads.append(
run_local_process(
"%sgenerate_tiles --local-process-number %i %s" %
(_get_path(), i, ' '.join([str(a)
for a in arguments]))))
else:
run_remote_process(
"%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene)
print('Tile generation started')
if options.shutdown:
run(options, 'sudo shutdown 0')
if options.wait and options.local:
while len(threads) > 0:
threads = [t for t in threads if t.is_alive()]
sleep(1)
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
}, "Tile generation controller")
def main() -> None:
"""Calculate the cost, main function."""
try:
parser = ArgumentParser(
description="Used to calculate the generation cost",
prog=sys.argv[0])
add_common_options(parser, tile_pyramid=False, dimensions=True)
parser.add_argument(
"--cost-algo",
"--calculate-cost-algorithm",
default="area",
dest="cost_algo",
choices=("area", "count"),
help=
"The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate.",
)
options = parser.parse_args()
gene = TileGeneration(
options.config,
options=options,
layer_name=options.layer,
base_config={"cost": {}},
multi_thread=False,
)
config = gene.get_config(options.config)
all_size: float = 0
tile_size: float = 0
all_tiles = 0
if options.layer:
layer = config.config["layers"][options.layer]
(all_size, all_time, all_price,
all_tiles) = _calculate_cost(gene, options.layer, options)
tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer_name in gene.get_config(
options.config).config["generation"]["default_layers"]:
print()
print(f"===== {layer_name} =====")
layer = config.config["layers"][layer_name]
gene.create_log_tiles_error(layer_name)
(size, time, price,
tiles) = _calculate_cost(gene, layer_name, options)
tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print()
print("===== GLOBAL =====")
print(f"Total number of tiles: {all_tiles}")
print(
f"Total generation time: {duration_format(all_time)} [d h:mm:ss]"
)
print(f"Total generation cost: {all_price:0.2f} [$]")
print()
s3_cost = all_size * gene.get_main_config(
).config["cost"]["s3"]["storage"] / (1024.0 * 1024 * 1024)
print(f"S3 Storage: {s3_cost:0.2f} [$/month]")
s3_get_cost = (
gene.get_main_config().config["cost"]["s3"]["get"] *
config.config["cost"]["request_per_layers"] / 10000.0 +
gene.get_main_config().config["cost"]["s3"]["download"] *
config.config["cost"]["request_per_layers"] * tile_size)
print(f"S3 get: {s3_get_cost:0.2f} [$/month]")
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] *
# gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] *
# gene.config['cost']['request_per_layers'] * tile_size)
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
sys.exit(1)
def _generate_legend_images(gene: TileGeneration) -> None:
assert gene.config_file
config = gene.get_config(gene.config_file)
cache = config.config["caches"][gene.options.cache]
for layer_name, layer in config.config["layers"].items():
if "legend_mime" in layer and "legend_extension" in layer:
if layer["type"] == "wms":
session = requests.session()
session.headers.update(layer["headers"])
previous_hash = None
for zoom, resolution in enumerate(
config.config["grids"][layer["grid"]]["resolutions"]):
legends = []
for wmslayer in layer["layers"].split(","):
response = session.get(layer["url"] + "?" + urlencode({
"SERVICE":
"WMS",
"VERSION":
layer.get("version", "1.0.0"),
"REQUEST":
"GetLegendGraphic",
"LAYER":
wmslayer,
"FORMAT":
layer["legend_mime"],
"TRANSPARENT":
"TRUE" if layer["legend_mime"] ==
"image/png" else "FALSE",
"STYLE":
layer["wmts_style"],
"SCALE":
resolution / 0.00028,
}))
try:
legends.append(
Image.open(BytesIO(response.content)))
except Exception: # pragma: nocover
logger.warning(
"Unable to read legend image for layer '%s'-'%s', resolution '%s': %s",
layer_name,
wmslayer,
resolution,
response.content,
exc_info=True,
)
width = max(i.size[0] for i in legends)
height = sum(i.size[1] for i in legends)
image = Image.new("RGBA", (width, height))
y = 0
for i in legends:
image.paste(i, (0, y))
y += i.size[1]
string_io = BytesIO()
image.save(string_io,
FORMAT_BY_CONTENT_TYPE[layer["legend_mime"]])
result = string_io.getvalue()
new_hash = sha1(result).hexdigest() # nosec
if new_hash != previous_hash:
previous_hash = new_hash
_send(
result,
f"1.0.0/{layer_name}/{layer['wmts_style']}/"
f"legend{zoom}.{layer['legend_extension']}",
layer["legend_mime"],
cache,
)
class Server:
def __init__(self, config_file):
self.filters = {}
self.max_zoom_seed = {}
logger.info("Config file: '{}'".format(config_file))
self.tilegeneration = TileGeneration(config_file)
self.expires_hours = self.tilegeneration.config['apache']['expires']
self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \
if 'static_allow_extension' in self.tilegeneration.config['server'] \
else ['jpeg', 'png', 'xml', 'js', 'html', 'css']
self.cache = self.tilegeneration.caches[
self.tilegeneration.config['server']['cache'] if 'cache' in
self.tilegeneration.config['server'] else self.tilegeneration.
config['generation']['default_cache']]
if self.cache['type'] == 's3': # pragma: no cover
client = tilecloud.store.s3.get_client(self.cache.get('host'))
bucket = self.cache['bucket']
def _get(self, path, **kwargs):
key_name = os.path.join('{folder}'.format(**self.cache), path)
try:
response = client.get_object(Bucket=bucket, Key=key_name)
return response['Body'].read(), response.get('ContentType')
except Exception:
client = tilecloud.store.s3.get_client(
self.cache.get('host'))
response = client.get_object(Bucket=bucket, Key=key_name)
return response['Body'].read(), response.get('ContentType')
else:
folder = self.cache['folder'] or ''
def _get(self, path, **kwargs):
if path.split(
'.'
)[-1] not in self.static_allow_extension: # pragma: no cover
return self.error(403, "Extension not allowed",
**kwargs), None
p = os.path.join(folder, path)
if not os.path.isfile(p): # pragma: no cover
return self.error(404, path + " not found", **kwargs), None
with open(p, 'rb') as file:
data = file.read()
mime = mimetypes.guess_type(p)
return data, mime[0]
# get capabilities or other static files
self._get = types.MethodType(_get, self)
mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \
'mapcache_base' in self.tilegeneration.config['server'] else \
'http://localhost/'
self.mapcache_baseurl = mapcache_base + self.tilegeneration.config[
'mapcache']['location'] + '/wmts'
self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \
'mapcache_headers' in self.tilegeneration.config['server'] else {}
geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \
'geoms_redirect' in self.tilegeneration.config['server'] else False
self.layers = self.tilegeneration.config['server']['layers'] if \
'layers' in self.tilegeneration.config['server'] else \
self.tilegeneration.layers.keys()
self.stores = {}
for layer_name in self.layers:
layer = self.tilegeneration.layers[layer_name]
# build geoms redirect
if geoms_redirect:
self.filters[
layer_name] = self.tilegeneration.get_geoms_filter(
layer=layer,
grid=layer['grid_ref'],
geoms=self.tilegeneration.get_geoms(
layer,
extent=layer['bbox']
if 'bbox' in layer else layer['grid_ref']['bbox'],
),
)
if 'min_resolution_seed' in layer:
max_zoom_seed = -1
for zoom, resolution in enumerate(
layer['grid_ref']['resolutions']):
if resolution > layer['min_resolution_seed']:
max_zoom_seed = zoom
self.max_zoom_seed[layer_name] = max_zoom_seed
else:
self.max_zoom_seed[layer_name] = 999999
# build stores
store_defs = [{
'ref': [layer_name],
'dimensions': {},
}]
for dimension in layer['dimensions']:
new_store_defs = []
for store_def in store_defs:
for value in dimension['values']:
dimensions = {}
dimensions.update(store_def['dimensions'])
dimensions[dimension['name']] = value
new_store_defs.append({
'ref': store_def['ref'] + [value],
'dimensions': dimensions,
})
store_defs = new_store_defs
for store_def in store_defs:
self.stores['/'.join(store_def['ref'])] = \
self.tilegeneration.get_store(self.cache, layer, read_only=True)
def __call__(self, environ, start_response):
params = {}
for key, value in parse_qs(environ['QUERY_STRING'], True).items():
params[key.upper()] = value[0]
path = None if len(params) > 0 else environ['PATH_INFO'][1:].split('/')
return self.serve(path, params, start_response=start_response)
def serve(self, path, params, **kwargs):
dimensions = []
metadata = {}
if path:
if len(path) >= 1 and path[0] == 'static':
body, mime = self._get('/'.join(path[1:]), **kwargs)
if mime is not None:
return self.response(
body, {
'Content-Type':
mime,
'Expires':
(datetime.datetime.utcnow() + datetime.timedelta(
hours=self.expires_hours)).isoformat(),
'Cache-Control':
"max-age={}".format((3600 * self.expires_hours)),
'Access-Control-Allow-Origin':
'*',
'Access-Control-Allow-Methods':
'GET',
}, **kwargs)
else: # pragma: no cover
return body
elif len(path) >= 1 and path[0] != 'wmts': # pragma: no cover
return self.error(
404,
"Type '{}' don't exists, allows values: 'wmts' or 'static'"
.format(path[0]), **kwargs)
path = path[1:] # remove type
if len(path) == 2 and path[0] == '1.0.0' and path[1].lower(
) == 'wmtscapabilities.xml':
params['SERVICE'] = 'WMTS'
params['VERSION'] = '1.0.0'
params['REQUEST'] = 'GetCapabilities'
elif len(path) < 7:
return self.error(400, "Not enough path", **kwargs)
else:
params['SERVICE'] = 'WMTS'
params['VERSION'] = path[0]
params['LAYER'] = path[1]
params['STYLE'] = path[2]
if params['LAYER'] in self.layers:
layer = self.tilegeneration.layers[params['LAYER']]
else:
return self.error(
400, "Wrong Layer '{}'".format(params['LAYER']),
**kwargs)
index = 3
dimensions = path[index:index + len(layer['dimensions'])]
for dimension in layer['dimensions']:
metadata["dimension_" + dimension['name']] = path[index]
params[dimension['name'].upper()] = path[index]
index += 1
last = path[-1].split('.')
if len(path) < index + 4: # pragma: no cover
return self.error(400, "Not enough path", **kwargs)
params['TILEMATRIXSET'] = path[index]
params['TILEMATRIX'] = path[index + 1]
params['TILEROW'] = path[index + 2]
if len(path) == index + 4:
params['REQUEST'] = 'GetTile'
params['TILECOL'] = last[0]
if last[1] != layer['extension']: # pragma: no cover
return self.error(
400, "Wrong extension '{}'".format(last[1]),
**kwargs)
elif len(path) == index + 6:
params['REQUEST'] = 'GetFeatureInfo'
params['TILECOL'] = path[index + 3]
params['I'] = path[index + 4]
params['J'] = last[0]
params['INFO_FORMAT'] = layer.get(
'info_formats', ['application/vnd.ogc.gml'])[0]
else: # pragma: no cover
return self.error(400, "Wrong path length", **kwargs)
params['FORMAT'] = layer['mime_type']
else:
if \
'SERVICE' not in params or \
'REQUEST' not in params or \
'VERSION' not in params:
return self.error(400,
"Not all required parameters are present",
**kwargs)
if params['SERVICE'] != 'WMTS':
return self.error(400,
"Wrong Service '{}'".format(params['SERVICE']),
**kwargs)
if params['VERSION'] != '1.0.0':
return self.error(400,
"Wrong Version '{}'".format(params['VERSION']),
**kwargs)
if params['REQUEST'] == 'GetCapabilities':
wmtscapabilities_file = self.cache['wmtscapabilities_file']
body, mime = self._get(wmtscapabilities_file, **kwargs)
if mime is not None:
return self.response(
body,
headers={
'Content-Type':
"application/xml",
'Expires':
(datetime.datetime.utcnow() + datetime.timedelta(
hours=self.expires_hours)).isoformat(),
'Cache-Control':
"max-age={}".format((3600 * self.expires_hours)),
'Access-Control-Allow-Origin':
'*',
'Access-Control-Allow-Methods':
'GET',
},
**kwargs)
else: # pragma: no cover
return body
if \
'FORMAT' not in params or \
'LAYER' not in params or \
'TILEMATRIXSET' not in params or \
'TILEMATRIX' not in params or \
'TILEROW' not in params or \
'TILECOL' not in params: # pragma: no cover
return self.error(400, "Not all required parameters are present",
**kwargs)
if not path:
if params['LAYER'] in self.layers:
layer = self.tilegeneration.layers[params['LAYER']]
else:
return self.error(400,
"Wrong Layer '{}'".format(params['LAYER']),
**kwargs)
for dimension in layer['dimensions']:
value = params[dimension['name'].upper()] \
if dimension['name'].upper() in params \
else dimension['default']
dimensions.append(value)
metadata["dimension_" + dimension['name']] = value
if params['STYLE'] != layer['wmts_style']:
return self.error(400, "Wrong Style '{}'".format(params['STYLE']),
**kwargs)
if params['TILEMATRIXSET'] != layer['grid']:
return self.error(
400,
"Wrong TileMatrixSet '{}'".format(params['TILEMATRIXSET']),
**kwargs)
tile = Tile(
TileCoord(
# TODO fix for matrix_identifier = resolution
int(params['TILEMATRIX']),
int(params['TILECOL']),
int(params['TILEROW']),
),
metadata=metadata)
if params['REQUEST'] == 'GetFeatureInfo':
if \
'I' not in params or \
'J' not in params or \
'INFO_FORMAT' not in params: # pragma: no cover
return self.error(400,
"Not all required parameters are present",
**kwargs)
if 'query_layers' in layer:
return self.forward(layer['url'] + '?' + urlencode(
{
'SERVICE': 'WMS',
'VERSION': layer['version'],
'REQUEST': 'GetFeatureInfo',
'LAYERS': layer['layers'],
'QUERY_LAYERS': layer['query_layers'],
'STYLES': params['STYLE'],
'FORMAT': params['FORMAT'],
'INFO_FORMAT': params['INFO_FORMAT'],
'WIDTH': layer['grid_ref']['tile_size'],
'HEIGHT': layer['grid_ref']['tile_size'],
'SRS': layer['grid_ref']['srs'],
'BBOX': layer['grid_ref']['obj'].extent(
tile.tilecoord),
'X': params['I'],
'Y': params['J'],
}),
no_cache=True,
**kwargs)
else: # pragma: no cover
return self.error(
400, "Layer '{}' not queryable".format(layer['name']),
**kwargs)
if params['REQUEST'] != 'GetTile':
return self.error(400,
"Wrong Request '{}'".format(params['REQUEST']),
**kwargs)
if params['FORMAT'] != layer['mime_type']:
return self.error(400,
"Wrong Format '{}'".format(params['FORMAT']),
**kwargs)
if tile.tilecoord.z > self.max_zoom_seed[
layer['name']]: # pragma: no cover
return self.forward(self.mapcache_baseurl + '?' +
urlencode(params),
headers=self.mapcache_header,
**kwargs)
if layer['name'] in self.filters:
layer_filter = self.filters[layer['name']]
meta_size = layer['meta_size']
meta_tilecoord = TileCoord(
# TODO fix for matrix_identifier = resolution
tile.tilecoord.z,
tile.tilecoord.x / meta_size * meta_size,
tile.tilecoord.y / meta_size * meta_size,
meta_size,
) if meta_size != 1 else tile.tilecoord
if not layer_filter.filter_tilecoord(
meta_tilecoord): # pragma: no cover
return self.forward(self.mapcache_baseurl + '?' +
urlencode(params),
headers=self.mapcache_header,
**kwargs)
store_ref = '/'.join([params['LAYER']] + list(dimensions))
if store_ref in self.stores: # pragma: no cover
store = self.stores[store_ref]
else: # pragma: no cover
return self.error(
400, "No store found for layer '{}' and dimensions {}".format(
layer['name'], ', '.join(dimensions)), **kwargs)
tile = store.get_one(tile)
if tile:
if tile.error:
return self.error(500, tile.error, **kwargs)
return self.response(
tile.data,
headers={
'Content-Type':
tile.content_type,
'Expires':
(datetime.datetime.utcnow() +
datetime.timedelta(hours=self.expires_hours)).isoformat(),
'Cache-Control':
"max-age={}".format((3600 * self.expires_hours)),
'Access-Control-Allow-Origin':
'*',
'Access-Control-Allow-Methods':
'GET',
},
**kwargs)
else:
return self.error(204, **kwargs)
def forward(self, url, headers=None, no_cache=False, **kwargs):
if headers is None:
headers = {}
if no_cache:
headers['Cache-Control'] = 'no-cache'
headers['Pragma'] = 'no-cache'
response = requests.get(url, headers=headers)
if response.status_code == 200:
response_headers = response.headers.copy()
if no_cache:
response_headers['Cache-Control'] = 'no-cache, no-store'
response_headers['Pragma'] = 'no-cache'
else: # pragma: no cover
response_headers['Expires'] = (
datetime.datetime.utcnow() +
datetime.timedelta(hours=self.expires_hours)).isoformat()
response_headers['Cache-Control'] = "max-age={}".format(
(3600 * self.expires_hours))
response_headers['Access-Control-Allow-Origin'] = '*'
response_headers['Access-Control-Allow-Methods'] = 'GET'
return self.response(response.content,
headers=response_headers,
**kwargs)
else: # pragma: no cover
message = "The URL '{}' return '{} {}', content:\n{}".format(
url, response.status_code, response.reason, response.text)
logger.warning(message)
return self.error(502, message=message, **kwargs)
HTTP_MESSAGES = {
204: '204 No Content',
400: '400 Bad Request',
403: '403 Forbidden',
404: '404 Not Found',
502: '502 Bad Gateway',
}
def error(self, code, message='', **kwargs):
kwargs['start_response'](self.HTTP_MESSAGES[code], [])
return [message]
@staticmethod
def response(data, headers=None, **kwargs):
if headers is None: # pragma: no cover
headers = {}
headers['Content-Length'] = str(len(data))
kwargs['start_response']('200 OK', headers.items())
return [data]
def _copy(
self,
options: Namespace,
gene: TileGeneration,
layer_name: str,
source: str,
dest: str,
task_name: str,
) -> None:
# disable metatiles
assert gene.config_file
config = gene.get_config(gene.config_file)
layer = config.config["layers"][layer_name]
cast(tilecloud_chain.configuration.LayerWms, layer)["meta"] = False
count_tiles_dropped = Count()
gene.create_log_tiles_error(layer_name)
source_tilestore = gene.get_tilesstore(source)
dest_tilestore = gene.get_tilesstore(dest)
gene.init_tilecoords(config, layer_name)
gene.add_geom_filter()
gene.add_logger()
gene.get(source_tilestore, "Get the tiles")
gene.imap(DropEmpty(gene))
# Discard tiles with certain content
if "empty_tile_detection" in layer:
empty_tile = layer["empty_tile_detection"]
gene.imap(
HashDropper(empty_tile["size"],
empty_tile["hash"],
store=dest_tilestore,
count=count_tiles_dropped))
if options.process:
gene.process(options.process)
gene.imap(DropEmpty(gene))
self.count = gene.counter_size()
gene.put(dest_tilestore, "Store the tiles")
gene.consume()
if not options.quiet:
print(f"""The tile {task_name} of layer '{layer_name}' is finish
Nb {task_name} tiles: {self.count.nb}
Nb errored tiles: {gene.error}
Nb dropped tiles: {count_tiles_dropped.nb}
Total time: {duration_format(gene.duration)}
Total size: {size_format(self.count.size)}
Time per tile: {(gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0} ms
Size per tile: {self.count.size / self.count.nb if self.count.nb != 0 else -1} o
""")
def main() -> None:
"""Run the tiles generation."""
try:
stats.init_backends({})
parser = ArgumentParser(description="Used to generate the tiles", prog=sys.argv[0])
add_common_options(parser, dimensions=True)
parser.add_argument(
"--get-hash", metavar="TILE", help="get the empty tiles hash, use the specified TILE z/x/y"
)
parser.add_argument(
"--get-bbox",
metavar="TILE",
help="get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles",
)
parser.add_argument(
"--role",
default="local",
choices=("local", "master", "slave"),
help="local/master/slave, master to file the queue and slave to generate the tiles",
)
parser.add_argument(
"--local-process-number", default=None, help="The number of process that we run in parallel"
)
parser.add_argument(
"--detach", default=False, action="store_true", help="run detached from the terminal"
)
parser.add_argument(
"--daemon", default=False, action="store_true", help="run continuously as a daemon"
)
parser.add_argument(
"--tiles",
metavar="FILE",
help="Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles",
)
options = parser.parse_args()
if options.detach:
detach()
gene = TileGeneration(
config_file=options.config, options=options, multi_thread=options.get_hash is None
)
if (
options.get_hash is None
and options.get_bbox is None
and options.config is not None
and "authorised_user" in gene.get_main_config().config.get("generation", {})
and gene.get_main_config().config["generation"]["authorised_user"] != getuser()
):
logger.error(
"not authorised, authorised user is: %s.",
gene.get_main_config().config["generation"]["authorised_user"],
)
sys.exit(1)
if options.config:
config = gene.get_config(options.config)
if options.cache is None and options.config:
options.cache = config.config["generation"]["default_cache"]
if options.tiles is not None and options.role not in ["local", "master"]:
logger.error("The --tiles option work only with role local or master")
sys.exit(1)
try:
generate = Generate(options, gene)
if options.role == "slave":
generate.gene()
elif options.layer:
generate.gene(options.layer)
elif options.get_bbox:
logger.error("With --get-bbox option you need to specify a layer")
sys.exit(1)
elif options.get_hash:
logger.error("With --get-hash option you need to specify a layer")
sys.exit(1)
else:
if options.config:
for layer in config.config["generation"].get(
"default_layers", config.config["layers"].keys()
):
generate.gene(layer)
except tilecloud.filter.error.TooManyErrors:
logger.exception("Too many errors")
sys.exit(1)
finally:
gene.close()
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
if os.environ.get("TESTS", "false").lower() == "true":
raise
sys.exit(1)
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog='./buildout/bin/generate_amazon')
add_comon_options(parser)
parser.add_argument('--deploy-config',
default=None,
dest="deploy_config",
metavar="FILE",
help='path to the deploy configuration file')
parser.add_argument('--status',
default=False,
action="store_true",
help='display the SQS queue status and exit')
parser.add_argument('--disable-geodata',
default=True,
action="store_false",
dest="geodata",
help='disable geodata synchronisation')
parser.add_argument('--disable-code',
default=True,
action="store_false",
dest="deploy_code",
help='disable deploy application code')
parser.add_argument('--disable-database',
default=True,
action="store_false",
dest="deploy_database",
help='disable deploy database')
parser.add_argument('--disable-fillqueue',
default=True,
action="store_false",
dest="fill_queue",
help='disable queue filling')
parser.add_argument('--disable-tilesgen',
default=True,
action="store_false",
dest="tiles_gen",
help='disable tile generation')
parser.add_argument('--host',
default=None,
help='The host used to generate tiles')
parser.add_argument('--shutdown',
default=False,
action="store_true",
help='Shut done the remote host after the task.')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print "EC2 not configured"
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if options.geodata and 'geodata_folder' in gene.config['ec2']:
print "==== Sync geodata ===="
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code:
print "==== Sync and build code ===="
error = gene.validate(gene.config['ec2'],
'ec2',
'code_folder',
required=True)
if error:
exit(1) # pragma: no cover
cmd = [
'rsync',
'--delete',
]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run_remote(cmd, host, project_dir, gene)
if 'apache_content' in gene.config[
'ec2'] and 'apache_config' in gene.config['ec2']:
run_remote(
'echo %s > %s' % (gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']), host,
project_dir, gene)
run_remote('sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir,
gene))
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(lambda x, y: x + y,
[timedelta(microseconds=int(r))
for r in times], timedelta()) / len(times)**2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print '==== Time results ===='
print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
print '''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)
if options.shutdown: # pragma: no cover
run_remote('sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue: # pragma: no cover
print "==== Till queue ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master'])
project_dir = gene.config['ec2']['code_folder']
run_remote(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene)
if options.tiles_gen: # pragma: no cover
print "==== Generate tiles ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'slave'])
arguments.append("--daemonize")
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir,
gene))
if options.shutdown:
for p in processes:
p.communicate() # wait process end
else:
print 'Tile generation started in background'
if options.shutdown:
run_remote('sudo shutdown 0')
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
}, "Tile generation controller")
def main():
stats.init_backends({})
parser = ArgumentParser(
description=
'Used to generate the contextual file like the capabilities, the legends, '
'the Apache and MapCache configuration',
prog=sys.argv[0])
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument('--status',
default=False,
action="store_true",
help='Display the SQS queue status and exit')
parser.add_argument('--capabilities',
'--generate-wmts-capabilities',
default=False,
action='store_true',
help='Generate the WMTS Capabilities')
parser.add_argument('--legends',
'--generate-legend-images',
default=False,
action='store_true',
dest='legends',
help='Generate the legend images')
parser.add_argument('--openlayers',
'--generate-openlayers-testpage',
default=False,
action='store_true',
dest='openlayers',
help='Generate openlayers test page')
parser.add_argument('--mapcache',
'--generate-mapcache-config',
default=False,
action='store_true',
dest='mapcache',
help='Generate MapCache configuration file')
parser.add_argument('--apache',
'--generate-apache-config',
default=False,
action='store_true',
dest='apache',
help='Generate Apache configuration file')
parser.add_argument(
'--dump-config',
default=False,
action='store_true',
help='Dump the used config with default values and exit')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(gene)
sys.exit(0)
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.dump_config:
for layer in gene.config['layers'].values():
gene.init_layer(layer, options)
_validate_generate_wmts_capabilities(gene.caches[options.cache], True)
for grid in gene.config['grids'].values():
if 'obj' in grid:
del grid['obj']
print(yaml.dump(gene.config))
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
def main():
parser = ArgumentParser(
description="Used to generate the contextual file like the capabilities, the legends, "
"the Apache and MapCache configuration",
prog=sys.argv[0],
)
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument(
"--capabilities",
"--generate-wmts-capabilities",
default=False,
action="store_true",
help="Generate the WMTS Capabilities",
)
parser.add_argument(
"--legends",
"--generate-legend-images",
default=False,
action="store_true",
dest="legends",
help="Generate the legend images",
)
parser.add_argument(
"--openlayers",
"--generate-openlayers-test-page",
default=False,
action="store_true",
dest="openlayers",
help="Generate openlayers test page",
)
parser.add_argument(
"--mapcache",
"--generate-mapcache-config",
default=False,
action="store_true",
dest="mapcache",
help="Generate MapCache configuration file",
)
parser.add_argument(
"--apache",
"--generate-apache-config",
default=False,
action="store_true",
dest="apache",
help="Generate Apache configuration file",
)
parser.add_argument(
"--dump-config", default=False, action="store_true", help="Dump the used config with default values and exit"
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.cache is None:
options.cache = gene.config["generation"]["default_cache"]
if options.dump_config:
for layer in gene.config["layers"].keys():
gene.set_layer(layer, options)
validate_calculate_cost(gene)
_validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
gene.validate_mapcache_config()
gene.validate_apache_config()
_validate_generate_openlayers(gene)
for grid in gene.config["grids"].values():
if "obj" in grid:
del grid["obj"]
print(yaml.dump(gene.config))
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
