def init_tilegeneration(config_file: Optional[str]) -> None:
"""Initialize the tile generation."""
global tilegeneration # pylint: disable=global-statement
if tilegeneration is None:
logger.info("Config file: '%s'", config_file)
log_level = os.environ.get("TILE_SERVER_LOGLEVEL")
tilegeneration = TileGeneration(
config_file,
collections.namedtuple( # type: ignore
"Options",
[
"verbose", "debug", "quiet", "bbox", "zoom", "test",
"near", "time", "geom", "ignore_error"
],
)(
log_level == "verbose", # type: ignore
log_level == "debug",
log_level == "quiet",
None,
None,
None,
None,
None,
True,
False,
),
configure_logging=False,
multi_thread=False,
maxconsecutive_errors=False,
)
def main():
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_comon_options(parser,
near=False,
time=False,
dimensions=True,
cache=False)
parser.add_argument("--process",
dest="process",
metavar="NAME",
help="The process name to do")
parser.add_argument("source", metavar="SOURCE", help="The source cache")
parser.add_argument("dest", metavar="DEST", help="The destination cache")
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if options.layer: # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.source, options.dest,
"copy")
else:
layers = (gene.config["generation"]["default_layers"]
if "default_layers" in gene.config["generation"] else
gene.config["layers"].keys())
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest,
"copy")
def main():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other',
prog=sys.argv[0])
add_comon_options(parser,
near=False,
time=False,
dimensions=True,
cache=False)
parser.add_argument('--process',
dest='process',
metavar="NAME",
help='The process name to do')
parser.add_argument('source', metavar="SOURCE", help='The source cache')
parser.add_argument('dest', metavar="DEST", help='The destination cache')
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if options.layer: # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.source, options.dest,
'copy')
else:
layers = gene.config['generation']['default_layers'] \
if 'default_layers' in gene.config['generation'] \
else gene.config['layers'].keys()
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest,
'copy')
def process() -> None:
"""Copy the tiles from a cache to an other."""
try:
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_common_options(parser, near=False, time=False, dimensions=True)
parser.add_argument("process",
metavar="PROCESS",
help="The process name to do")
options = parser.parse_args()
gene = TileGeneration(options.config, options, multi_thread=False)
copy = Copy()
if options.layer:
copy.copy(options, gene, options.layer, options.cache,
options.cache, "process")
else:
assert gene.config_file
config = gene.get_config(gene.config_file)
layers_name = (config.config["generation"]["default_layers"]
if "default_layers" in config.config.get(
"generation", {}) else
config.config["layers"].keys())
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache,
"process")
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
sys.exit(1)
def main():
parser = ArgumentParser(
description='Used to calculate the generation cost',
prog=sys.argv[0]
)
add_comon_options(parser, tile_pyramid=False)
parser.add_argument(
'--cost-algo', '--calculate-cost-algorithm', default='area', dest='cost_algo',
choices=('area', 'count'),
help="The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate."
)
options = parser.parse_args()
gene = TileGeneration(
options.config, options,
layer_name=options.layer, base_config={'cost': {}}
)
all_size = 0
tile_size = 0
all_tiles = 0
if (options.layer):
(all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options)
tile_size = gene.layer['cost']['tile_size'] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer in gene.config['generation']['default_layers']:
print("")
print("===== %s =====" % layer)
gene.set_layer(layer, options)
(size, time, price, tiles) = _calculate_cost(gene, options)
tile_size += gene.layer['cost']['tile_size'] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print("")
print("===== GLOBAL =====")
print("Total number of tiles: %i" % all_tiles)
print('Total generation time: %s [d h:mm:ss]' % (duration_format(all_time)))
print('Total generation cost: %0.2f [$]' % all_price)
print("")
print('S3 Storage: %0.2f [$/month]' % (all_size * gene.config['cost']['s3']['storage'] / (1024.0 * 1024 * 1024)))
print('S3 get: %0.2f [$/month]' % (
gene.config['cost']['s3']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
gene.config['cost']['s3']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
)
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
if 'ec2' in gene.config:
print('ESB storage: %0.2f [$/month]' % (
gene.config['cost']['esb']['storage'] * gene.config['cost']['esb_size'])
)
sys.exit(0)
def test_validate_type(self):
class Opt:
quiet = False
verbose = False
debug = False
test = 0
zoom = None
gene = TileGeneration('tilegeneration/test.yaml', Opt())
obj = {'value': 1}
self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
self.assertEquals(obj['value'], 1)
obj = {'value': 1.0}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': '1 + 1'}
self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
self.assertEquals(obj['value'], 2)
obj = {'value': '1 * 1.5'}
self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
self.assertEquals(obj['value'], 2)
obj = {'value': 'a'}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': {}}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': []}
self.assertEquals(gene.validate(obj, 'object', 'value', int), True)
obj = {'value': 1}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 1.0)
obj = {'value': 1.0}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 1.0)
obj = {'value': '1 + 1'}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 2.0)
obj = {'value': '1 * 1.5'}
self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
self.assertEquals(obj['value'], 1.5)
obj = {'value': 'a'}
self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
obj = {'value': {}}
self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
obj = {'value': []}
self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
def test_near(self):
class Opt:
zoom = '3'
test = 196
near = [600000.0, 200000.0]
quiet = False
verbose = False
debug = False
time = None
logging_configuration_file = None
gene = TileGeneration('tilecloud_chain/tests/tilegeneration/test.yaml', Opt(), 'point')
self.assertEqual(gene.geoms[3].bounds, (583840.0, 173360.0, 624800.0, 214320.0))
def main() -> None:
"""Copy the tiles from a cache to an other."""
try:
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_common_options(parser,
near=False,
time=False,
dimensions=True,
cache=False)
parser.add_argument("--process",
dest="process",
metavar="NAME",
help="The process name to do")
parser.add_argument("source",
metavar="SOURCE",
help="The source cache")
parser.add_argument("dest",
metavar="DEST",
help="The destination cache")
options = parser.parse_args()
gene = TileGeneration(options.config, options)
assert gene.config_file
config = gene.get_config(gene.config_file)
if options.layer:
copy = Copy()
copy.copy(options, gene, options.layer, options.source,
options.dest, "copy")
else:
layers = (config.config["generation"]["default_layers"]
if "default_layers" in config.config["generation"] else
config.config["layers"].keys())
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest,
"copy")
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
if os.environ.get("TESTS", "false").lower() == "true":
raise
sys.exit(1)
def process():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other',
prog='./buildout/bin/generate_copy')
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument('process',
metavar="PROCESS",
help='The process name to do')
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if (options.layer): # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.cache, options.cache,
'process')
else:
for layer in gene.config['generation']['default_layers']:
copy = Copy()
copy.copy(options, gene, layer, options.cache, options.cache,
'process')
def process():
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument("process",
metavar="PROCESS",
help="The process name to do")
options = parser.parse_args()
gene = TileGeneration(options.config, options, multi_thread=False)
copy = Copy()
if options.layer: # pragma: no cover
copy.copy(options, gene, options.layer, options.cache, options.cache,
"process")
else:
layers_name = (gene.config["generation"]["default_layers"]
if "default_layers" in gene.config.get(
"generation", {}) else gene.layers.keys())
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache,
"process")
def process():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other',
prog=sys.argv[0])
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument('process',
metavar="PROCESS",
help='The process name to do')
options = parser.parse_args()
gene = TileGeneration(options.config, options)
copy = Copy()
if options.layer: # pragma: no cover
copy.copy(options, gene, options.layer, options.cache, options.cache,
'process')
else:
layers_name = gene.config['generation']['default_layers'] \
if 'default_layers' in gene.config.get('generation', {}) \
else gene.layers.keys()
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache,
'process')
def init_tilegeneration(config_file):
global tilegeneration # pylint: disable=global-statement
if tilegeneration is None:
logger.info("Config file: '%s'", config_file)
log_level = os.environ.get("TILE_SERVER_LOGLEVEL")
tilegeneration = TileGeneration(
config_file,
collections.namedtuple(
"Options",
["verbose", "debug", "quiet", "bbox", "zoom", "test", "near", "time", "geom"],
)(
log_level == "verbose",
log_level == "debug",
log_level == "quiet",
None,
None,
None,
None,
None,
True,
),
configure_logging=False,
multi_thread=False,
)
def __init__(self, config_file):
self.filters = {}
self.max_zoom_seed = {}
logger.info("Config file: '{}'".format(config_file))
self.tilegeneration = TileGeneration(config_file)
self.expires_hours = self.tilegeneration.config['apache']['expires']
self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \
if 'static_allow_extension' in self.tilegeneration.config['server'] \
else ['jpeg', 'png', 'xml', 'js', 'html', 'css']
self.cache = self.tilegeneration.caches[
self.tilegeneration.config['server']['cache'] if 'cache' in
self.tilegeneration.config['server'] else self.tilegeneration.
config['generation']['default_cache']]
if self.cache['type'] == 's3': # pragma: no cover
client = tilecloud.store.s3.get_client(self.cache.get('host'))
bucket = self.cache['bucket']
def _get(self, path, **kwargs):
key_name = os.path.join('{folder}'.format(**self.cache), path)
try:
response = client.get_object(Bucket=bucket, Key=key_name)
return response['Body'].read(), response.get('ContentType')
except Exception:
client = tilecloud.store.s3.get_client(
self.cache.get('host'))
response = client.get_object(Bucket=bucket, Key=key_name)
return response['Body'].read(), response.get('ContentType')
else:
folder = self.cache['folder'] or ''
def _get(self, path, **kwargs):
if path.split(
'.'
)[-1] not in self.static_allow_extension: # pragma: no cover
return self.error(403, "Extension not allowed",
**kwargs), None
p = os.path.join(folder, path)
if not os.path.isfile(p): # pragma: no cover
return self.error(404, path + " not found", **kwargs), None
with open(p, 'rb') as file:
data = file.read()
mime = mimetypes.guess_type(p)
return data, mime[0]
# get capabilities or other static files
self._get = types.MethodType(_get, self)
mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \
'mapcache_base' in self.tilegeneration.config['server'] else \
'http://localhost/'
self.mapcache_baseurl = mapcache_base + self.tilegeneration.config[
'mapcache']['location'] + '/wmts'
self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \
'mapcache_headers' in self.tilegeneration.config['server'] else {}
geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \
'geoms_redirect' in self.tilegeneration.config['server'] else False
self.layers = self.tilegeneration.config['server']['layers'] if \
'layers' in self.tilegeneration.config['server'] else \
self.tilegeneration.layers.keys()
self.stores = {}
for layer_name in self.layers:
layer = self.tilegeneration.layers[layer_name]
# build geoms redirect
if geoms_redirect:
self.filters[
layer_name] = self.tilegeneration.get_geoms_filter(
layer=layer,
grid=layer['grid_ref'],
geoms=self.tilegeneration.get_geoms(
layer,
extent=layer['bbox']
if 'bbox' in layer else layer['grid_ref']['bbox'],
),
)
if 'min_resolution_seed' in layer:
max_zoom_seed = -1
for zoom, resolution in enumerate(
layer['grid_ref']['resolutions']):
if resolution > layer['min_resolution_seed']:
max_zoom_seed = zoom
self.max_zoom_seed[layer_name] = max_zoom_seed
else:
self.max_zoom_seed[layer_name] = 999999
# build stores
store_defs = [{
'ref': [layer_name],
'dimensions': {},
}]
for dimension in layer['dimensions']:
new_store_defs = []
for store_def in store_defs:
for value in dimension['values']:
dimensions = {}
dimensions.update(store_def['dimensions'])
dimensions[dimension['name']] = value
new_store_defs.append({
'ref': store_def['ref'] + [value],
'dimensions': dimensions,
})
store_defs = new_store_defs
for store_def in store_defs:
self.stores['/'.join(store_def['ref'])] = \
self.tilegeneration.get_store(self.cache, layer, read_only=True)
def main() -> None:
"""Calculate the cost, main function."""
try:
parser = ArgumentParser(
description="Used to calculate the generation cost",
prog=sys.argv[0])
add_common_options(parser, tile_pyramid=False, dimensions=True)
parser.add_argument(
"--cost-algo",
"--calculate-cost-algorithm",
default="area",
dest="cost_algo",
choices=("area", "count"),
help=
"The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate.",
)
options = parser.parse_args()
gene = TileGeneration(
options.config,
options=options,
layer_name=options.layer,
base_config={"cost": {}},
multi_thread=False,
)
config = gene.get_config(options.config)
all_size: float = 0
tile_size: float = 0
all_tiles = 0
if options.layer:
layer = config.config["layers"][options.layer]
(all_size, all_time, all_price,
all_tiles) = _calculate_cost(gene, options.layer, options)
tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer_name in gene.get_config(
options.config).config["generation"]["default_layers"]:
print()
print(f"===== {layer_name} =====")
layer = config.config["layers"][layer_name]
gene.create_log_tiles_error(layer_name)
(size, time, price,
tiles) = _calculate_cost(gene, layer_name, options)
tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print()
print("===== GLOBAL =====")
print(f"Total number of tiles: {all_tiles}")
print(
f"Total generation time: {duration_format(all_time)} [d h:mm:ss]"
)
print(f"Total generation cost: {all_price:0.2f} [$]")
print()
s3_cost = all_size * gene.get_main_config(
).config["cost"]["s3"]["storage"] / (1024.0 * 1024 * 1024)
print(f"S3 Storage: {s3_cost:0.2f} [$/month]")
s3_get_cost = (
gene.get_main_config().config["cost"]["s3"]["get"] *
config.config["cost"]["request_per_layers"] / 10000.0 +
gene.get_main_config().config["cost"]["s3"]["download"] *
config.config["cost"]["request_per_layers"] * tile_size)
print(f"S3 get: {s3_get_cost:0.2f} [$/month]")
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] *
# gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] *
# gene.config['cost']['request_per_layers'] * tile_size)
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
sys.exit(1)
def main():
parser = ArgumentParser(description='Used to generate the tiles',
prog='./buildout/bin/generate_tiles')
add_comon_options(parser, dimensions=True)
parser.add_argument(
'--get-hash',
metavar="TILE",
help='get the empty tiles hash, use the specified TILE z/x/y')
parser.add_argument(
'--get-bbox',
metavar="TILE",
help=
'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--role',
default='local',
choices=('local', 'master', 'slave'),
help='local/master/slave, master to file the queue and '
'slave to generate the tiles')
parser.add_argument('--daemonize',
default=False,
action="store_true",
help='run as a daemon')
parser.add_argument(
'--tiles',
metavar="FILE",
help=
'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument('--generated-tiles-file',
metavar="FILE",
help='Store the tiles in a file (unrecommended)')
options = parser.parse_args()
if options.daemonize:
daemonize() # pragma: no cover
gene = TileGeneration(options.config, options)
if options.get_hash is None and options.get_bbox is None and \
'authorised_user' in gene.config['generation'] and \
gene.config['generation']['authorised_user'] != getuser():
exit('not authorised, authorised user is: %s.' %
gene.config['generation']['authorised_user'])
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.tiles is not None and options.role not in [
'local', 'master'
]: # pragma: no cover
exit("The --tiles option work only with role local or master")
try:
if (options.layer):
generate = Generate()
generate.gene(options, gene, options.layer)
elif options.get_bbox: # pragma: no cover
exit("With --get-bbox option we needs to specify a layer")
elif options.get_hash: # pragma: no cover
exit("With --get-hash option we needs to specify a layer")
elif options.tiles: # pragma: no cover
exit("With --tiles option we needs to specify a layer")
else:
for layer in gene.config['generation']['default_layers']:
generate = Generate()
generate.gene(options, gene, layer)
finally:
if gene.error_file is not None:
gene.error_file.close()
def main():
stats.init_backends({})
parser = ArgumentParser(description="Used to generate the tiles",
prog=sys.argv[0])
add_comon_options(parser, dimensions=True)
parser.add_argument(
"--get-hash",
metavar="TILE",
help="get the empty tiles hash, use the specified TILE z/x/y")
parser.add_argument(
"--get-bbox",
metavar="TILE",
help=
"get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles",
)
parser.add_argument(
"--role",
default="local",
choices=("local", "master", "slave"),
help=
"local/master/slave, master to file the queue and slave to generate the tiles",
)
parser.add_argument("--local-process-number",
default=None,
help="The number of process that we run in parallel")
parser.add_argument("--detach",
default=False,
action="store_true",
help="run detached from the terminal")
parser.add_argument("--daemon",
default=False,
action="store_true",
help="run continuously as a daemon")
parser.add_argument(
"--tiles",
metavar="FILE",
help=
"Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles",
)
options = parser.parse_args()
if options.detach:
detach() # pragma: no cover
gene = TileGeneration(options.config,
options,
multi_thread=options.get_hash is None)
if (options.get_hash is None and options.get_bbox is None
and "authorised_user" in gene.config["generation"]
and gene.config["generation"]["authorised_user"] != getuser()):
sys.exit("not authorised, authorised user is: {}.".format(
gene.config["generation"]["authorised_user"]))
if options.cache is None:
options.cache = gene.config["generation"]["default_cache"]
if options.tiles is not None and options.role not in [
"local", "master"
]: # pragma: no cover
sys.exit("The --tiles option work only with role local or master")
try:
generate = Generate(options, gene)
if options.role == "slave":
generate.gene()
elif options.layer:
generate.gene(gene.layers[options.layer])
elif options.get_bbox: # pragma: no cover
sys.exit("With --get-bbox option you need to specify a layer")
elif options.get_hash: # pragma: no cover
sys.exit("With --get-hash option you need to specify a layer")
elif options.tiles: # pragma: no cover
sys.exit("With --tiles option you need to specify a layer")
else:
for layer in gene.config["generation"].get("default_layers",
gene.layers.keys()):
generate.gene(gene.layers[layer])
finally:
gene.close()
def main():
parser = ArgumentParser(
description="Used to calculate the generation cost", prog=sys.argv[0])
add_comon_options(parser, tile_pyramid=False, dimensions=True)
parser.add_argument(
"--cost-algo",
"--calculate-cost-algorithm",
default="area",
dest="cost_algo",
choices=("area", "count"),
help=
"The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate.",
)
options = parser.parse_args()
gene = TileGeneration(options.config,
options,
layer_name=options.layer,
base_config={"cost": {}},
multi_thread=False)
all_size = 0
tile_size = 0
all_tiles = 0
if options.layer:
layer = gene.layers[options.layer]
(all_size, all_time, all_price,
all_tiles) = _calculate_cost(gene, layer, options)
tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer_name in gene.config["generation"]["default_layers"]:
print("")
print("===== {} =====".format(layer_name))
layer = gene.layers[layer_name]
gene.init_layer(layer, options)
(size, time, price, tiles) = _calculate_cost(gene, layer, options)
tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print("")
print("===== GLOBAL =====")
print("Total number of tiles: {}".format(all_tiles))
print("Total generation time: {} [d h:mm:ss]".format(
(duration_format(all_time))))
print("Total generation cost: {0:0.2f} [$]".format(all_price))
print("")
print("S3 Storage: {0:0.2f} [$/month]".format(
all_size * gene.config["cost"]["s3"]["storage"] /
(1024.0 * 1024 * 1024)))
print("S3 get: {0:0.2f} [$/month]".format(
(gene.config["cost"]["s3"]["get"] *
gene.config["cost"]["request_per_layers"] / 10000.0 +
gene.config["cost"]["s3"]["download"] *
gene.config["cost"]["request_per_layers"] * tile_size)))
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] *
# gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] *
# gene.config['cost']['request_per_layers'] * tile_size)
sys.exit(0)
def main() -> None:
"""Generate the contextual file like the legends."""
try:
stats.init_backends({})
parser = ArgumentParser(
description=
"Used to generate the contextual file like the capabilities, the legends, "
"the OpenLayers example",
prog=sys.argv[0],
)
add_common_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument("--status",
default=False,
action="store_true",
help="Display the SQS queue status and exit")
parser.add_argument(
"--legends",
"--generate-legend-images",
default=False,
action="store_true",
dest="legends",
help="Generate the legend images",
)
parser.add_argument(
"--dump-config",
default=False,
action="store_true",
help="Dump the used config with default values and exit",
)
options = parser.parse_args()
gene = TileGeneration(options.config,
options,
layer_name=options.layer)
assert gene.config_file
config = gene.get_config(gene.config_file)
if options.status:
status(gene)
sys.exit(0)
if options.cache is None:
options.cache = config.config["generation"]["default_cache"]
if options.dump_config:
_validate_generate_wmts_capabilities(
config.config["caches"][options.cache], options.cache, True)
yaml = ruamel.yaml.YAML() # type: ignore
out = StringIO()
yaml.dump(config.config, out)
print(out.getvalue())
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
if os.environ.get("TESTS", "false").lower() == "true":
raise
sys.exit(1)
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog='./buildout/bin/generate_amazon')
add_comon_options(parser)
parser.add_argument('--deploy-config',
default=None,
dest="deploy_config",
metavar="FILE",
help='path to the deploy configuration file')
parser.add_argument('--status',
default=False,
action="store_true",
help='display the SQS queue status and exit')
parser.add_argument('--disable-geodata',
default=True,
action="store_false",
dest="geodata",
help='disable geodata synchronisation')
parser.add_argument('--disable-code',
default=True,
action="store_false",
dest="deploy_code",
help='disable deploy application code')
parser.add_argument('--disable-database',
default=True,
action="store_false",
dest="deploy_database",
help='disable deploy database')
parser.add_argument('--disable-fillqueue',
default=True,
action="store_false",
dest="fill_queue",
help='disable queue filling')
parser.add_argument('--disable-tilesgen',
default=True,
action="store_false",
dest="tiles_gen",
help='disable tile generation')
parser.add_argument('--host',
default=None,
help='The host used to generate tiles')
parser.add_argument('--shutdown',
default=False,
action="store_true",
help='Shut done the remote host after the task.')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print "EC2 not configured"
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if options.geodata and 'geodata_folder' in gene.config['ec2']:
print "==== Sync geodata ===="
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code:
print "==== Sync and build code ===="
error = gene.validate(gene.config['ec2'],
'ec2',
'code_folder',
required=True)
if error:
exit(1) # pragma: no cover
cmd = [
'rsync',
'--delete',
]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run_remote(cmd, host, project_dir, gene)
if 'apache_content' in gene.config[
'ec2'] and 'apache_config' in gene.config['ec2']:
run_remote(
'echo %s > %s' % (gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']), host,
project_dir, gene)
run_remote('sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir,
gene))
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(lambda x, y: x + y,
[timedelta(microseconds=int(r))
for r in times], timedelta()) / len(times)**2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print '==== Time results ===='
print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
print '''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)
if options.shutdown: # pragma: no cover
run_remote('sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue: # pragma: no cover
print "==== Till queue ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master'])
project_dir = gene.config['ec2']['code_folder']
run_remote(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene)
if options.tiles_gen: # pragma: no cover
print "==== Generate tiles ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'slave'])
arguments.append("--daemonize")
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir,
gene))
if options.shutdown:
for p in processes:
p.communicate() # wait process end
else:
print 'Tile generation started in background'
if options.shutdown:
run_remote('sudo shutdown 0')
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
}, "Tile generation controller")
def main():
stats.init_backends({})
parser = ArgumentParser(
description="Used to generate the contextual file like the capabilities, the legends, "
"the Apache and MapCache configuration",
prog=sys.argv[0],
)
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument(
"--status", default=False, action="store_true", help="Display the SQS queue status and exit"
)
parser.add_argument(
"--capabilities",
"--generate-wmts-capabilities",
default=False,
action="store_true",
help="Generate the WMTS Capabilities",
)
parser.add_argument(
"--legends",
"--generate-legend-images",
default=False,
action="store_true",
dest="legends",
help="Generate the legend images",
)
parser.add_argument(
"--openlayers",
"--generate-openlayers-testpage",
default=False,
action="store_true",
dest="openlayers",
help="Generate openlayers test page",
)
parser.add_argument(
"--mapcache",
"--generate-mapcache-config",
default=False,
action="store_true",
dest="mapcache",
help="Generate MapCache configuration file",
)
parser.add_argument(
"--mapcache-version", default="1.4", choices=("1.4", "1.6"), help="The used version of MapCache"
)
parser.add_argument(
"--apache",
"--generate-apache-config",
default=False,
action="store_true",
dest="apache",
help="Generate Apache configuration file",
)
parser.add_argument(
"--dump-config",
default=False,
action="store_true",
help="Dump the used config with default values and exit",
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(gene)
sys.exit(0)
if options.cache is None:
options.cache = gene.config["generation"]["default_cache"]
if options.dump_config:
for layer in gene.config["layers"].values():
gene.init_layer(layer, options)
_validate_generate_wmts_capabilities(gene.caches[options.cache], True)
for grid in gene.config["grids"].values():
if "obj" in grid:
del grid["obj"]
print(yaml.dump(gene.config))
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene, options.mapcache_version)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
def main() -> None:
"""Run the tiles generation."""
try:
stats.init_backends({})
parser = ArgumentParser(description="Used to generate the tiles", prog=sys.argv[0])
add_common_options(parser, dimensions=True)
parser.add_argument(
"--get-hash", metavar="TILE", help="get the empty tiles hash, use the specified TILE z/x/y"
)
parser.add_argument(
"--get-bbox",
metavar="TILE",
help="get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles",
)
parser.add_argument(
"--role",
default="local",
choices=("local", "master", "slave"),
help="local/master/slave, master to file the queue and slave to generate the tiles",
)
parser.add_argument(
"--local-process-number", default=None, help="The number of process that we run in parallel"
)
parser.add_argument(
"--detach", default=False, action="store_true", help="run detached from the terminal"
)
parser.add_argument(
"--daemon", default=False, action="store_true", help="run continuously as a daemon"
)
parser.add_argument(
"--tiles",
metavar="FILE",
help="Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles",
)
options = parser.parse_args()
if options.detach:
detach()
gene = TileGeneration(
config_file=options.config, options=options, multi_thread=options.get_hash is None
)
if (
options.get_hash is None
and options.get_bbox is None
and options.config is not None
and "authorised_user" in gene.get_main_config().config.get("generation", {})
and gene.get_main_config().config["generation"]["authorised_user"] != getuser()
):
logger.error(
"not authorised, authorised user is: %s.",
gene.get_main_config().config["generation"]["authorised_user"],
)
sys.exit(1)
if options.config:
config = gene.get_config(options.config)
if options.cache is None and options.config:
options.cache = config.config["generation"]["default_cache"]
if options.tiles is not None and options.role not in ["local", "master"]:
logger.error("The --tiles option work only with role local or master")
sys.exit(1)
try:
generate = Generate(options, gene)
if options.role == "slave":
generate.gene()
elif options.layer:
generate.gene(options.layer)
elif options.get_bbox:
logger.error("With --get-bbox option you need to specify a layer")
sys.exit(1)
elif options.get_hash:
logger.error("With --get-hash option you need to specify a layer")
sys.exit(1)
else:
if options.config:
for layer in config.config["generation"].get(
"default_layers", config.config["layers"].keys()
):
generate.gene(layer)
except tilecloud.filter.error.TooManyErrors:
logger.exception("Too many errors")
sys.exit(1)
finally:
gene.close()
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("Exit with exception")
if os.environ.get("TESTS", "false").lower() == "true":
raise
sys.exit(1)
def __init__(self, config_file):
self.filters = {}
self.max_zoom_seed = {}
logger.info("Config file: '%s'" % config_file)
self.tilegeneration = TileGeneration(config_file)
if not self.tilegeneration.validate_apache_config(): # pragma: no cover
raise "Apache configuration error"
self.expires_hours = self.tilegeneration.config['apache']['expires']
self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \
if 'static_allow_extension' in self.tilegeneration.config['server'] \
else ['jpeg', 'png', 'xml', 'js', 'html', 'css']
self.cache = self.tilegeneration.caches[
self.tilegeneration.config['server']['cache'] if
'cache' in self.tilegeneration.config['server'] else
self.tilegeneration.config['generation']['default_cache']
]
if self.cache['type'] == 's3': # pragma: no cover
s3bucket = S3Connection().bucket(self.cache['bucket'])
def _get(self, path, **kwargs):
global s3bucket
try:
s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path))
responce = s3key.get()
return responce.body, responce.headers['Content-Type']
except:
s3bucket = S3Connection().bucket(self.cache['bucket'])
s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path))
responce = s3key.get()
return responce.body, responce.headers['Content-Type']
else:
folder = self.cache['folder'] or ''
def _get(self, path, **kwargs):
if path.split('.')[-1] not in self.static_allow_extension: # pragma: no cover
return self.error(403, "Extension not allowed", **kwargs), None
p = os.path.join(folder, path)
if not os.path.isfile(p): # pragma: no cover
return self.error(404, path + " not found", **kwargs), None
with open(p, 'rb') as file:
data = file.read()
mime = mimetypes.guess_type(p)
return data, mime[0]
# get capabilities or other static files
self._get = types.MethodType(_get, self)
if not self.tilegeneration.validate_mapcache_config(): # pragma: no cover
raise "Mapcache configuration error"
mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \
'mapcache_base' in self.tilegeneration.config['server'] else \
'http://localhost/'
self.mapcache_baseurl = mapcache_base + self.tilegeneration.config['mapcache']['location'] + '/wmts'
self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \
'mapcache_headers' in self.tilegeneration.config['server'] else {}
geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \
'geoms_redirect' in self.tilegeneration.config['server'] else False
self.layers = self.tilegeneration.config['server']['layers'] if \
'layers' in self.tilegeneration.config['server'] else \
self.tilegeneration.layers.keys()
self.stores = {}
for layer_name in self.layers:
layer = self.tilegeneration.layers[layer_name]
# build geoms redirect
if geoms_redirect:
self.filters[layer_name] = self.tilegeneration.get_geoms_filter(
layer=layer,
grid=layer['grid_ref'],
geoms=self.tilegeneration.get_geoms(
layer,
extent=layer['bbox'] if 'bbox' in layer else layer['grid_ref']['bbox'],
),
)
if 'min_resolution_seed' in layer:
max_zoom_seed = -1
for zoom, resolution in enumerate(layer['grid_ref']['resolutions']):
if resolution > layer['min_resolution_seed']:
max_zoom_seed = zoom
self.max_zoom_seed[layer_name] = max_zoom_seed
else:
self.max_zoom_seed[layer_name] = sys.maxint
# build stores
store_defs = [{
'ref': [layer_name],
'dimensions': [],
}]
for dimension in layer['dimensions']:
new_store_defs = []
for store_def in store_defs:
for value in dimension['values']:
new_store_defs.append({
'ref': store_def['ref'] + [value],
'dimensions': store_def['dimensions'] + [(dimension['name'], value)],
})
store_defs = new_store_defs
for store_def in store_defs:
self.stores['/'.join(store_def['ref'])] = \
self.tilegeneration.get_store(self.cache, layer, store_def['dimensions'], read_only=True)
def main():
parser = ArgumentParser(description='Used to generate the tiles',
prog=sys.argv[0])
add_comon_options(parser, dimensions=True)
parser.add_argument(
'--get-hash',
metavar="TILE",
help='get the empty tiles hash, use the specified TILE z/x/y')
parser.add_argument(
'--get-bbox',
metavar="TILE",
help=
'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--role',
default='local',
choices=('local', 'master', 'slave'),
help='local/master/slave, master to file the queue and '
'slave to generate the tiles')
parser.add_argument("--local-process-number",
default=None,
help="The number of process that we run in parallel")
parser.add_argument('--detach',
default=False,
action="store_true",
help='run detached from the terminal')
parser.add_argument('--daemon',
default=False,
action="store_true",
help='run continuously as a daemon')
parser.add_argument(
'--tiles',
metavar="FILE",
help=
'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument('--generated-tiles-file',
metavar="FILE",
help='Store the tiles in a file (unrecommended)')
options = parser.parse_args()
if options.detach:
detach() # pragma: no cover
gene = TileGeneration(options.config, options)
if options.get_hash is None and options.get_bbox is None and \
'authorised_user' in gene.config['generation'] and \
gene.config['generation']['authorised_user'] != getuser():
exit('not authorised, authorised user is: {}.'.format(
gene.config['generation']['authorised_user']))
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.tiles is not None and options.role not in [
'local', 'master'
]: # pragma: no cover
exit("The --tiles option work only with role local or master")
try:
if options.role == 'slave':
generate = Generate()
generate.gene(options, gene)
elif (options.layer):
generate = Generate()
generate.gene(options, gene, options.layer)
elif options.get_bbox: # pragma: no cover
exit("With --get-bbox option we needs to specify a layer")
elif options.get_hash: # pragma: no cover
exit("With --get-hash option we needs to specify a layer")
elif options.tiles: # pragma: no cover
exit("With --tiles option we needs to specify a layer")
else:
for layer in gene.config['generation'].get('default_layers',
gene.layers.keys()):
generate = Generate()
generate.gene(options, gene, layer)
finally:
if gene.error_file is not None:
gene.error_file.close()
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog=sys.argv[0])
add_comon_options(parser)
parser.add_argument('--deploy-config',
default=None,
dest="deploy_config",
metavar="FILE",
help='path to the deploy configuration file')
parser.add_argument('--status',
default=False,
action="store_true",
help='display the SQS queue status and exit')
parser.add_argument('--disable-geodata',
default=True,
action="store_false",
dest="geodata",
help='disable geodata synchronisation')
parser.add_argument('--disable-code',
default=True,
action="store_false",
dest="deploy_code",
help='disable deploy application code')
parser.add_argument('--disable-database',
default=True,
action="store_false",
dest="deploy_database",
help='disable deploy database')
parser.add_argument('--disable-fillqueue',
default=True,
action="store_false",
dest="fill_queue",
help='disable queue filling')
parser.add_argument('--disable-tilesgen',
default=True,
action="store_false",
dest="tiles_gen",
help='disable tile generation')
parser.add_argument('--host',
default=None,
help='The host used to generate tiles')
parser.add_argument('--shutdown',
default=False,
action="store_true",
help='Shut done the remote host after the task.')
parser.add_argument('--wait',
default=False,
action="store_true",
help='Wait that all the tasks will finish.')
parser.add_argument('--local',
default=False,
action="store_true",
help='Run the generation locally')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print("EC2 not configured")
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if not options.local and options.geodata and 'geodata_folder' in gene.config[
'ec2']: # pragma: no cover
print("==== Sync geodata ====")
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code and not options.local:
print("==== Sync and build code ====")
cmd = [
'rsync',
'--delete',
]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run(options, cmd % environ, host, project_dir, gene)
if 'apache_content' in gene.config[
'ec2'] and 'apache_config' in gene.config['ec2']:
run(
options,
'echo %s > %s' % (gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']), host,
project_dir, gene)
run(options, 'sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database and not options.local:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata and not options.local:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = None if options.local else gene.config['ec2'][
'code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
"%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene))
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
if PY3:
results = [r.decode('utf-8') for r in results]
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(lambda x, y: x + y,
[timedelta(microseconds=int(r))
for r in times], timedelta()) / len(times)**2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print('==== Time results ====')
print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
print('''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))
if options.shutdown: # pragma: no cover
run(options, 'sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue and not options.local: # pragma: no cover
print("==== Till queue ====")
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master', '--quiet'])
project_dir = gene.config['ec2']['code_folder']
run_remote_process(
options, "%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene)
sleep(5)
attributes = gene.get_sqs_queue().get_attributes()
print("\rTiles to generate: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
))
if options.tiles_gen: # pragma: no cover
print("==== Generate tiles ====")
if options.wait and not options.local:
print("")
class Status(Thread):
def run(self): # pragma: no cover
while True:
attributes = gene.get_sqs_queue().get_attributes()
print("\rTiles to generate/generating: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
))
sleep(1)
status_thread = Status()
status_thread.setDaemon(True)
status_thread.start()
arguments = _get_arguments(options)
arguments.extend(['--quiet'])
if not options.local:
arguments.extend(['--role', 'slave'])
project_dir = None if options.local else gene.config['ec2'][
'code_folder']
threads = []
for i in range(gene.config['ec2']['number_process']):
if options.local:
threads.append(
run_local_process(
"%sgenerate_tiles --local-process-number %i %s" %
(_get_path(), i, ' '.join([str(a)
for a in arguments]))))
else:
run_remote_process(
"%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene)
print('Tile generation started')
if options.shutdown:
run(options, 'sudo shutdown 0')
if options.wait and options.local:
while len(threads) > 0:
threads = [t for t in threads if t.is_alive()]
sleep(1)
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
}, "Tile generation controller")
def main():
stats.init_backends({})
parser = ArgumentParser(
description=
'Used to generate the contextual file like the capabilities, the legends, '
'the Apache and MapCache configuration',
prog=sys.argv[0])
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument('--status',
default=False,
action="store_true",
help='Display the SQS queue status and exit')
parser.add_argument('--capabilities',
'--generate-wmts-capabilities',
default=False,
action='store_true',
help='Generate the WMTS Capabilities')
parser.add_argument('--legends',
'--generate-legend-images',
default=False,
action='store_true',
dest='legends',
help='Generate the legend images')
parser.add_argument('--openlayers',
'--generate-openlayers-testpage',
default=False,
action='store_true',
dest='openlayers',
help='Generate openlayers test page')
parser.add_argument('--mapcache',
'--generate-mapcache-config',
default=False,
action='store_true',
dest='mapcache',
help='Generate MapCache configuration file')
parser.add_argument('--apache',
'--generate-apache-config',
default=False,
action='store_true',
dest='apache',
help='Generate Apache configuration file')
parser.add_argument(
'--dump-config',
default=False,
action='store_true',
help='Dump the used config with default values and exit')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(gene)
sys.exit(0)
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.dump_config:
for layer in gene.config['layers'].values():
gene.init_layer(layer, options)
_validate_generate_wmts_capabilities(gene.caches[options.cache], True)
for grid in gene.config['grids'].values():
if 'obj' in grid:
del grid['obj']
print(yaml.dump(gene.config))
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
def main():
parser = ArgumentParser(
description=
'Used to generate the contextual file like the capabilities, the legends, '
'the Apache and MapCache configuration',
prog='./buildout/bin/generate_controller')
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument('--capabilities',
'--generate_wmts_capabilities',
default=False,
action="store_true",
help='Generate the WMTS Capabilities')
parser.add_argument('--legends',
'--generate_legend_images',
default=False,
action="store_true",
dest='legends',
help='Generate the legend images')
parser.add_argument('--openlayers',
'--generate-openlayers-test-page',
default=False,
action="store_true",
dest='openlayers',
help='Generate openlayers test page')
parser.add_argument('--mapcache',
'--generate-mapcache-config',
default=False,
action="store_true",
dest='mapcache',
help='Generate MapCache configuration file')
parser.add_argument('--apache',
'--generate-apache-config',
default=False,
action="store_true",
dest='apache',
help='Generate Apache configuration file')
parser.add_argument(
'--dump-config',
default=False,
action="store_true",
help='Dump the used config with default values and exit')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.dump_config:
for layer in gene.config['layers'].keys():
gene.set_layer(layer, options)
validate_calculate_cost(gene)
_validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
gene.validate_mapcache_config()
gene.validate_apache_config()
_validate_generate_openlayers(gene)
for grid in gene.config['grids'].values():
if 'obj' in grid:
del grid['obj']
print yaml.dump(gene.config)
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
