def main():
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_comon_options(parser,
near=False,
time=False,
dimensions=True,
cache=False)
parser.add_argument("--process",
dest="process",
metavar="NAME",
help="The process name to do")
parser.add_argument("source", metavar="SOURCE", help="The source cache")
parser.add_argument("dest", metavar="DEST", help="The destination cache")
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if options.layer: # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.source, options.dest,
"copy")
else:
layers = (gene.config["generation"]["default_layers"]
if "default_layers" in gene.config["generation"] else
gene.config["layers"].keys())
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest,
"copy")
def main():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other',
prog=sys.argv[0])
add_comon_options(parser,
near=False,
time=False,
dimensions=True,
cache=False)
parser.add_argument('--process',
dest='process',
metavar="NAME",
help='The process name to do')
parser.add_argument('source', metavar="SOURCE", help='The source cache')
parser.add_argument('dest', metavar="DEST", help='The destination cache')
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if options.layer: # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.source, options.dest,
'copy')
else:
layers = gene.config['generation']['default_layers'] \
if 'default_layers' in gene.config['generation'] \
else gene.config['layers'].keys()
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest,
'copy')
def main():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other', prog=sys.argv[0]
)
add_comon_options(parser, near=False, time=False, dimensions=True, cache=False)
parser.add_argument(
'--process', dest='process', metavar="NAME",
help='The process name to do'
)
parser.add_argument(
'source',
metavar="SOURCE",
help='The source cache'
)
parser.add_argument(
'dest',
metavar="DEST",
help='The destination cache'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if (options.layer): # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.source, options.dest, 'copy')
else:
layers = gene.config['generation']['default_layers'] \
if 'default_layers' in gene.config['generation'] \
else gene.config['layers'].keys()
for layer in layers:
copy = Copy()
copy.copy(options, gene, layer, options.source, options.dest, 'copy')
def main():
parser = ArgumentParser(
description='Used to calculate the generation cost',
prog=sys.argv[0]
)
add_comon_options(parser, tile_pyramid=False)
parser.add_argument(
'--cost-algo', '--calculate-cost-algorithm', default='area', dest='cost_algo',
choices=('area', 'count'),
help="The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate."
)
options = parser.parse_args()
gene = TileGeneration(
options.config, options,
layer_name=options.layer, base_config={'cost': {}}
)
all_size = 0
tile_size = 0
all_tiles = 0
if (options.layer):
(all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options)
tile_size = gene.layer['cost']['tile_size'] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer in gene.config['generation']['default_layers']:
print("")
print("===== %s =====" % layer)
gene.set_layer(layer, options)
(size, time, price, tiles) = _calculate_cost(gene, options)
tile_size += gene.layer['cost']['tile_size'] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print("")
print("===== GLOBAL =====")
print("Total number of tiles: %i" % all_tiles)
print('Total generation time: %s [d h:mm:ss]' % (duration_format(all_time)))
print('Total generation cost: %0.2f [$]' % all_price)
print("")
print('S3 Storage: %0.2f [$/month]' % (all_size * gene.config['cost']['s3']['storage'] / (1024.0 * 1024 * 1024)))
print('S3 get: %0.2f [$/month]' % (
gene.config['cost']['s3']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
gene.config['cost']['s3']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
)
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
if 'ec2' in gene.config:
print('ESB storage: %0.2f [$/month]' % (
gene.config['cost']['esb']['storage'] * gene.config['cost']['esb_size'])
)
sys.exit(0)
def process():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other', prog='./buildout/bin/generate_copy'
)
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument(
'process', metavar="PROCESS",
help='The process name to do'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if (options.layer): # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.cache, options.cache, 'process')
else:
for layer in gene.config['generation']['default_layers']:
copy = Copy()
copy.copy(options, gene, layer, options.cache, options.cache, 'process')
def process():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other',
prog='./buildout/bin/generate_copy')
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument('process',
metavar="PROCESS",
help='The process name to do')
options = parser.parse_args()
gene = TileGeneration(options.config, options)
if (options.layer): # pragma: no cover
copy = Copy()
copy.copy(options, gene, options.layer, options.cache, options.cache,
'process')
else:
for layer in gene.config['generation']['default_layers']:
copy = Copy()
copy.copy(options, gene, layer, options.cache, options.cache,
'process')
def process():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other', prog=sys.argv[0]
)
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument(
'process', metavar="PROCESS",
help='The process name to do'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options)
copy = Copy()
if (options.layer): # pragma: no cover
copy.copy(options, gene, options.layer, options.cache, options.cache, 'process')
else:
layers_name = gene.config['generation']['default_layers'] \
if 'default_layers' in gene.config.get('generation', {}) \
else gene.layers.keys()
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache, 'process')
def process():
parser = ArgumentParser(
description='Used to copy the tiles from a cache to an other',
prog=sys.argv[0])
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument('process',
metavar="PROCESS",
help='The process name to do')
options = parser.parse_args()
gene = TileGeneration(options.config, options)
copy = Copy()
if options.layer: # pragma: no cover
copy.copy(options, gene, options.layer, options.cache, options.cache,
'process')
else:
layers_name = gene.config['generation']['default_layers'] \
if 'default_layers' in gene.config.get('generation', {}) \
else gene.layers.keys()
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache,
'process')
def process():
parser = ArgumentParser(
description="Used to copy the tiles from a cache to an other",
prog=sys.argv[0])
add_comon_options(parser, near=False, time=False, dimensions=True)
parser.add_argument("process",
metavar="PROCESS",
help="The process name to do")
options = parser.parse_args()
gene = TileGeneration(options.config, options, multi_thread=False)
copy = Copy()
if options.layer: # pragma: no cover
copy.copy(options, gene, options.layer, options.cache, options.cache,
"process")
else:
layers_name = (gene.config["generation"]["default_layers"]
if "default_layers" in gene.config.get(
"generation", {}) else gene.layers.keys())
for layer in layers_name:
copy.copy(options, gene, layer, options.cache, options.cache,
"process")
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog='./buildout/bin/generate_amazon')
add_comon_options(parser)
parser.add_argument('--deploy-config',
default=None,
dest="deploy_config",
metavar="FILE",
help='path to the deploy configuration file')
parser.add_argument('--status',
default=False,
action="store_true",
help='display the SQS queue status and exit')
parser.add_argument('--disable-geodata',
default=True,
action="store_false",
dest="geodata",
help='disable geodata synchronisation')
parser.add_argument('--disable-code',
default=True,
action="store_false",
dest="deploy_code",
help='disable deploy application code')
parser.add_argument('--disable-database',
default=True,
action="store_false",
dest="deploy_database",
help='disable deploy database')
parser.add_argument('--disable-fillqueue',
default=True,
action="store_false",
dest="fill_queue",
help='disable queue filling')
parser.add_argument('--disable-tilesgen',
default=True,
action="store_false",
dest="tiles_gen",
help='disable tile generation')
parser.add_argument('--host',
default=None,
help='The host used to generate tiles')
parser.add_argument('--shutdown',
default=False,
action="store_true",
help='Shut done the remote host after the task.')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print "EC2 not configured"
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if options.geodata and 'geodata_folder' in gene.config['ec2']:
print "==== Sync geodata ===="
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code:
print "==== Sync and build code ===="
error = gene.validate(gene.config['ec2'],
'ec2',
'code_folder',
required=True)
if error:
exit(1) # pragma: no cover
cmd = [
'rsync',
'--delete',
]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run_remote(cmd, host, project_dir, gene)
if 'apache_content' in gene.config[
'ec2'] and 'apache_config' in gene.config['ec2']:
run_remote(
'echo %s > %s' % (gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']), host,
project_dir, gene)
run_remote('sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir,
gene))
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(lambda x, y: x + y,
[timedelta(microseconds=int(r))
for r in times], timedelta()) / len(times)**2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print '==== Time results ===='
print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
print '''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)
if options.shutdown: # pragma: no cover
run_remote('sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue: # pragma: no cover
print "==== Till queue ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master'])
project_dir = gene.config['ec2']['code_folder']
run_remote(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene)
if options.tiles_gen: # pragma: no cover
print "==== Generate tiles ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'slave'])
arguments.append("--daemonize")
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir,
gene))
if options.shutdown:
for p in processes:
p.communicate() # wait process end
else:
print 'Tile generation started in background'
if options.shutdown:
run_remote('sudo shutdown 0')
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
}, "Tile generation controller")
def main():
stats.init_backends({})
parser = ArgumentParser(
description="Used to generate the contextual file like the capabilities, the legends, "
"the Apache and MapCache configuration",
prog=sys.argv[0],
)
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument(
"--status", default=False, action="store_true", help="Display the SQS queue status and exit"
)
parser.add_argument(
"--capabilities",
"--generate-wmts-capabilities",
default=False,
action="store_true",
help="Generate the WMTS Capabilities",
)
parser.add_argument(
"--legends",
"--generate-legend-images",
default=False,
action="store_true",
dest="legends",
help="Generate the legend images",
)
parser.add_argument(
"--openlayers",
"--generate-openlayers-testpage",
default=False,
action="store_true",
dest="openlayers",
help="Generate openlayers test page",
)
parser.add_argument(
"--mapcache",
"--generate-mapcache-config",
default=False,
action="store_true",
dest="mapcache",
help="Generate MapCache configuration file",
)
parser.add_argument(
"--mapcache-version", default="1.4", choices=("1.4", "1.6"), help="The used version of MapCache"
)
parser.add_argument(
"--apache",
"--generate-apache-config",
default=False,
action="store_true",
dest="apache",
help="Generate Apache configuration file",
)
parser.add_argument(
"--dump-config",
default=False,
action="store_true",
help="Dump the used config with default values and exit",
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(gene)
sys.exit(0)
if options.cache is None:
options.cache = gene.config["generation"]["default_cache"]
if options.dump_config:
for layer in gene.config["layers"].values():
gene.init_layer(layer, options)
_validate_generate_wmts_capabilities(gene.caches[options.cache], True)
for grid in gene.config["grids"].values():
if "obj" in grid:
del grid["obj"]
print(yaml.dump(gene.config))
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene, options.mapcache_version)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
def main():
parser = ArgumentParser(description='Used to generate the tiles',
prog='./buildout/bin/generate_tiles')
add_comon_options(parser, dimensions=True)
parser.add_argument(
'--get-hash',
metavar="TILE",
help='get the empty tiles hash, use the specified TILE z/x/y')
parser.add_argument(
'--get-bbox',
metavar="TILE",
help=
'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--role',
default='local',
choices=('local', 'master', 'slave'),
help='local/master/slave, master to file the queue and '
'slave to generate the tiles')
parser.add_argument('--daemonize',
default=False,
action="store_true",
help='run as a daemon')
parser.add_argument(
'--tiles',
metavar="FILE",
help=
'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument('--generated-tiles-file',
metavar="FILE",
help='Store the tiles in a file (unrecommended)')
options = parser.parse_args()
if options.daemonize:
daemonize() # pragma: no cover
gene = TileGeneration(options.config, options)
if options.get_hash is None and options.get_bbox is None and \
'authorised_user' in gene.config['generation'] and \
gene.config['generation']['authorised_user'] != getuser():
exit('not authorised, authorised user is: %s.' %
gene.config['generation']['authorised_user'])
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.tiles is not None and options.role not in [
'local', 'master'
]: # pragma: no cover
exit("The --tiles option work only with role local or master")
try:
if (options.layer):
generate = Generate()
generate.gene(options, gene, options.layer)
elif options.get_bbox: # pragma: no cover
exit("With --get-bbox option we needs to specify a layer")
elif options.get_hash: # pragma: no cover
exit("With --get-hash option we needs to specify a layer")
elif options.tiles: # pragma: no cover
exit("With --tiles option we needs to specify a layer")
else:
for layer in gene.config['generation']['default_layers']:
generate = Generate()
generate.gene(options, gene, layer)
finally:
if gene.error_file is not None:
gene.error_file.close()
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog='./buildout/bin/generate_amazon'
)
add_comon_options(parser)
parser.add_argument(
'--deploy-config', default=None, dest="deploy_config", metavar="FILE",
help='path to the deploy configuration file'
)
parser.add_argument(
'--status', default=False, action="store_true",
help='display the SQS queue status and exit'
)
parser.add_argument(
'--disable-geodata', default=True, action="store_false", dest="geodata",
help='disable geodata synchronisation'
)
parser.add_argument(
'--disable-code', default=True, action="store_false", dest="deploy_code",
help='disable deploy application code'
)
parser.add_argument(
'--disable-database', default=True, action="store_false", dest="deploy_database",
help='disable deploy database'
)
parser.add_argument(
'--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
help='disable queue filling'
)
parser.add_argument(
'--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
help='disable tile generation'
)
parser.add_argument(
'--host', default=None,
help='The host used to generate tiles'
)
parser.add_argument(
'--shutdown', default=False, action="store_true",
help='Shut done the remote host after the task.'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print "EC2 not configured"
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if options.geodata and 'geodata_folder' in gene.config['ec2']:
print "==== Sync geodata ===="
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options,
'-r', gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code:
print "==== Sync and build code ===="
error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
if error:
exit(1) # pragma: no cover
cmd = ['rsync', '--delete', ]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run_remote(cmd, host, project_dir, gene)
if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
run_remote(
'echo %s > %s' % (
gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']
), host, project_dir, gene
)
run_remote('sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene
)
)
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(
lambda x, y: x + y,
[timedelta(microseconds=int(r)) for r in times],
timedelta()
) / len(times) ** 2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print '==== Time results ===='
print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
print '''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)
if options.shutdown: # pragma: no cover
run_remote('sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue: # pragma: no cover
print "==== Till queue ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master'])
project_dir = gene.config['ec2']['code_folder']
run_remote(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene
)
if options.tiles_gen: # pragma: no cover
print "==== Generate tiles ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'slave'])
arguments.append("--daemonize")
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene)
)
if options.shutdown:
for p in processes:
p.communicate() # wait process end
else:
print 'Tile generation started in background'
if options.shutdown:
run_remote('sudo shutdown 0')
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'],
"""The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
{
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
},
"Tile generation controller")
def main():
parser = ArgumentParser(
description='Used to generate the contextual file like the capabilities, the legends, '
'the Apache and MapCache configuration',
prog='./buildout/bin/generate_controller'
)
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument(
'--capabilities', '--generate_wmts_capabilities', default=False, action="store_true",
help='Generate the WMTS Capabilities'
)
parser.add_argument(
'--legends', '--generate_legend_images', default=False, action="store_true", dest='legends',
help='Generate the legend images'
)
parser.add_argument(
'--openlayers', '--generate-openlayers-test-page', default=False,
action="store_true", dest='openlayers',
help='Generate openlayers test page'
)
parser.add_argument(
'--mapcache', '--generate-mapcache-config', default=False, action="store_true", dest='mapcache',
help='Generate MapCache configuration file'
)
parser.add_argument(
'--apache', '--generate-apache-config', default=False, action="store_true", dest='apache',
help='Generate Apache configuration file'
)
parser.add_argument(
'--dump-config', default=False, action="store_true",
help='Dump the used config with default values and exit'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.dump_config:
for layer in gene.config['layers'].keys():
gene.set_layer(layer, options)
validate_calculate_cost(gene)
_validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
gene.validate_mapcache_config()
gene.validate_apache_config()
_validate_generate_openlayers(gene)
for grid in gene.config['grids'].values():
if 'obj' in grid:
del grid['obj']
print yaml.dump(gene.config)
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog=sys.argv[0]
)
add_comon_options(parser)
parser.add_argument(
'--deploy-config', default=None, dest="deploy_config", metavar="FILE",
help='path to the deploy configuration file'
)
parser.add_argument(
'--status', default=False, action="store_true",
help='display the SQS queue status and exit'
)
parser.add_argument(
'--disable-geodata', default=True, action="store_false", dest="geodata",
help='disable geodata synchronisation'
)
parser.add_argument(
'--disable-code', default=True, action="store_false", dest="deploy_code",
help='disable deploy application code'
)
parser.add_argument(
'--disable-database', default=True, action="store_false", dest="deploy_database",
help='disable deploy database'
)
parser.add_argument(
'--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
help='disable queue filling'
)
parser.add_argument(
'--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
help='disable tile generation'
)
parser.add_argument(
'--host', default=None,
help='The host used to generate tiles'
)
parser.add_argument(
'--shutdown', default=False, action="store_true",
help='Shut done the remote host after the task.'
)
parser.add_argument(
'--wait', default=False, action="store_true",
help='Wait that all the tasks will finish.'
)
parser.add_argument(
'--local', default=False, action="store_true",
help='Run the generation locally'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print("EC2 not configured")
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']:
print("==== Sync geodata ====")
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options,
'-r', gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code and not options.local:
print("==== Sync and build code ====")
error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
if error:
exit(1) # pragma: no cover
cmd = ['rsync', '--delete', ]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run(options, cmd % environ, host, project_dir, gene)
if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
run(
options,
'echo %s > %s' % (
gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']
), host, project_dir, gene
)
run(options, 'sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database and not options.local:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata and not options.local:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = None if options.local else gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
"%sgenerate_tiles %s" % (
_get_path(),
' '.join([str(a) for a in arguments])
), host, project_dir, gene
)
)
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
if PY3:
results = [r.decode('utf-8') for r in results]
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(
lambda x, y: x + y,
[timedelta(microseconds=int(r)) for r in times],
timedelta()
) / len(times) ** 2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print('==== Time results ====')
print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
print('''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))
if options.shutdown: # pragma: no cover
run(options, 'sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue and not options.local: # pragma: no cover
print("==== Till queue ====")
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master', '--quiet'])
project_dir = gene.config['ec2']['code_folder']
run_remote_process(
options,
"%sgenerate_tiles %s" % (
_get_path(),
' '.join([str(a) for a in arguments])
), host, project_dir, gene
)
sleep(5)
attributes = gene.get_sqs_queue().get_attributes()
print(
"\rTiles to generate: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
)
)
if options.tiles_gen: # pragma: no cover
print("==== Generate tiles ====")
if options.wait and not options.local:
print("")
class Status(Thread):
def run(self): # pragma: no cover
while True:
attributes = gene.get_sqs_queue().get_attributes()
print(
"\rTiles to generate/generating: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
)
)
sleep(1)
status_thread = Status()
status_thread.setDaemon(True)
status_thread.start()
arguments = _get_arguments(options)
arguments.extend(['--quiet'])
if not options.local:
arguments.extend(['--role', 'slave'])
project_dir = None if options.local else gene.config['ec2']['code_folder']
threads = []
for i in range(gene.config['ec2']['number_process']):
if options.local:
threads.append(run_local_process(
"%sgenerate_tiles --local-process-number %i %s" % (
_get_path(),
i, ' '.join([str(a) for a in arguments])
)
))
else:
run_remote_process(
"%sgenerate_tiles %s" % (
_get_path(),
' '.join([str(a) for a in arguments])
), host, project_dir, gene
)
print('Tile generation started')
if options.shutdown:
run(options, 'sudo shutdown 0')
if options.wait and options.local:
while len(threads) > 0:
threads = [t for t in threads if t.is_alive()]
sleep(1)
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'],
"""The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
{
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
},
"Tile generation controller"
)
def main():
parser = ArgumentParser(
description=
'Used to generate the contextual file like the capabilities, the legends, '
'the Apache and MapCache configuration',
prog='./buildout/bin/generate_controller')
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument('--capabilities',
'--generate_wmts_capabilities',
default=False,
action="store_true",
help='Generate the WMTS Capabilities')
parser.add_argument('--legends',
'--generate_legend_images',
default=False,
action="store_true",
dest='legends',
help='Generate the legend images')
parser.add_argument('--openlayers',
'--generate-openlayers-test-page',
default=False,
action="store_true",
dest='openlayers',
help='Generate openlayers test page')
parser.add_argument('--mapcache',
'--generate-mapcache-config',
default=False,
action="store_true",
dest='mapcache',
help='Generate MapCache configuration file')
parser.add_argument('--apache',
'--generate-apache-config',
default=False,
action="store_true",
dest='apache',
help='Generate Apache configuration file')
parser.add_argument(
'--dump-config',
default=False,
action="store_true",
help='Dump the used config with default values and exit')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.dump_config:
for layer in gene.config['layers'].keys():
gene.set_layer(layer, options)
validate_calculate_cost(gene)
_validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
gene.validate_mapcache_config()
gene.validate_apache_config()
_validate_generate_openlayers(gene)
for grid in gene.config['grids'].values():
if 'obj' in grid:
del grid['obj']
print yaml.dump(gene.config)
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
def main():
parser = ArgumentParser(description='Used to generate the tiles', prog=sys.argv[0])
add_comon_options(parser, dimensions=True)
parser.add_argument(
'--get-hash', metavar="TILE",
help='get the empty tiles hash, use the specified TILE z/x/y'
)
parser.add_argument(
'--get-bbox', metavar="TILE",
help='get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--role', default='local', choices=('local', 'master', 'slave'),
help='local/master/slave, master to file the queue and '
'slave to generate the tiles'
)
parser.add_argument(
"--local-process-number", default=None,
help="The number of process that we run in parallel"
)
parser.add_argument(
'--daemonize', default=False, action="store_true",
help='run as a daemon'
)
parser.add_argument(
'--tiles', metavar="FILE",
help='Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--generated-tiles-file', metavar="FILE",
help='Store the tiles in a file (unrecommended)'
)
options = parser.parse_args()
if options.daemonize:
daemonize() # pragma: no cover
gene = TileGeneration(options.config, options)
if options.get_hash is None and options.get_bbox is None and \
'authorised_user' in gene.config['generation'] and \
gene.config['generation']['authorised_user'] != getuser():
exit('not authorised, authorised user is: %s.' % gene.config['generation']['authorised_user'])
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.tiles is not None and options.role not in ['local', 'master']: # pragma: no cover
exit("The --tiles option work only with role local or master")
try:
if (options.layer):
generate = Generate()
generate.gene(options, gene, options.layer)
elif options.get_bbox: # pragma: no cover
exit("With --get-bbox option we needs to specify a layer")
elif options.get_hash: # pragma: no cover
exit("With --get-hash option we needs to specify a layer")
elif options.tiles: # pragma: no cover
exit("With --tiles option we needs to specify a layer")
else:
for layer in gene.config['generation'].get('default_layers', gene.layers.keys()):
generate = Generate()
generate.gene(options, gene, layer)
finally:
if gene.error_file is not None:
gene.error_file.close()
def main():
parser = ArgumentParser(description='Used to generate the tiles',
prog=sys.argv[0])
add_comon_options(parser, dimensions=True)
parser.add_argument(
'--get-hash',
metavar="TILE",
help='get the empty tiles hash, use the specified TILE z/x/y')
parser.add_argument(
'--get-bbox',
metavar="TILE",
help=
'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--role',
default='local',
choices=('local', 'master', 'slave'),
help='local/master/slave, master to file the queue and '
'slave to generate the tiles')
parser.add_argument("--local-process-number",
default=None,
help="The number of process that we run in parallel")
parser.add_argument('--detach',
default=False,
action="store_true",
help='run detached from the terminal')
parser.add_argument('--daemon',
default=False,
action="store_true",
help='run continuously as a daemon')
parser.add_argument(
'--tiles',
metavar="FILE",
help=
'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument('--generated-tiles-file',
metavar="FILE",
help='Store the tiles in a file (unrecommended)')
options = parser.parse_args()
if options.detach:
detach() # pragma: no cover
gene = TileGeneration(options.config, options)
if options.get_hash is None and options.get_bbox is None and \
'authorised_user' in gene.config['generation'] and \
gene.config['generation']['authorised_user'] != getuser():
exit('not authorised, authorised user is: {}.'.format(
gene.config['generation']['authorised_user']))
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.tiles is not None and options.role not in [
'local', 'master'
]: # pragma: no cover
exit("The --tiles option work only with role local or master")
try:
if options.role == 'slave':
generate = Generate()
generate.gene(options, gene)
elif (options.layer):
generate = Generate()
generate.gene(options, gene, options.layer)
elif options.get_bbox: # pragma: no cover
exit("With --get-bbox option we needs to specify a layer")
elif options.get_hash: # pragma: no cover
exit("With --get-hash option we needs to specify a layer")
elif options.tiles: # pragma: no cover
exit("With --tiles option we needs to specify a layer")
else:
for layer in gene.config['generation'].get('default_layers',
gene.layers.keys()):
generate = Generate()
generate.gene(options, gene, layer)
finally:
if gene.error_file is not None:
gene.error_file.close()
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog=sys.argv[0])
add_comon_options(parser)
parser.add_argument('--deploy-config',
default=None,
dest="deploy_config",
metavar="FILE",
help='path to the deploy configuration file')
parser.add_argument('--status',
default=False,
action="store_true",
help='display the SQS queue status and exit')
parser.add_argument('--disable-geodata',
default=True,
action="store_false",
dest="geodata",
help='disable geodata synchronisation')
parser.add_argument('--disable-code',
default=True,
action="store_false",
dest="deploy_code",
help='disable deploy application code')
parser.add_argument('--disable-database',
default=True,
action="store_false",
dest="deploy_database",
help='disable deploy database')
parser.add_argument('--disable-fillqueue',
default=True,
action="store_false",
dest="fill_queue",
help='disable queue filling')
parser.add_argument('--disable-tilesgen',
default=True,
action="store_false",
dest="tiles_gen",
help='disable tile generation')
parser.add_argument('--host',
default=None,
help='The host used to generate tiles')
parser.add_argument('--shutdown',
default=False,
action="store_true",
help='Shut done the remote host after the task.')
parser.add_argument('--wait',
default=False,
action="store_true",
help='Wait that all the tasks will finish.')
parser.add_argument('--local',
default=False,
action="store_true",
help='Run the generation locally')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print("EC2 not configured")
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if not options.local and options.geodata and 'geodata_folder' in gene.config[
'ec2']: # pragma: no cover
print("==== Sync geodata ====")
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code and not options.local:
print("==== Sync and build code ====")
cmd = [
'rsync',
'--delete',
]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run(options, cmd % environ, host, project_dir, gene)
if 'apache_content' in gene.config[
'ec2'] and 'apache_config' in gene.config['ec2']:
run(
options,
'echo %s > %s' % (gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']), host,
project_dir, gene)
run(options, 'sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database and not options.local:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata and not options.local:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = None if options.local else gene.config['ec2'][
'code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
"%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene))
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
if PY3:
results = [r.decode('utf-8') for r in results]
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(lambda x, y: x + y,
[timedelta(microseconds=int(r))
for r in times], timedelta()) / len(times)**2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print('==== Time results ====')
print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
print('''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))
if options.shutdown: # pragma: no cover
run(options, 'sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue and not options.local: # pragma: no cover
print("==== Till queue ====")
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master', '--quiet'])
project_dir = gene.config['ec2']['code_folder']
run_remote_process(
options, "%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene)
sleep(5)
attributes = gene.get_sqs_queue().get_attributes()
print("\rTiles to generate: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
))
if options.tiles_gen: # pragma: no cover
print("==== Generate tiles ====")
if options.wait and not options.local:
print("")
class Status(Thread):
def run(self): # pragma: no cover
while True:
attributes = gene.get_sqs_queue().get_attributes()
print("\rTiles to generate/generating: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
))
sleep(1)
status_thread = Status()
status_thread.setDaemon(True)
status_thread.start()
arguments = _get_arguments(options)
arguments.extend(['--quiet'])
if not options.local:
arguments.extend(['--role', 'slave'])
project_dir = None if options.local else gene.config['ec2'][
'code_folder']
threads = []
for i in range(gene.config['ec2']['number_process']):
if options.local:
threads.append(
run_local_process(
"%sgenerate_tiles --local-process-number %i %s" %
(_get_path(), i, ' '.join([str(a)
for a in arguments]))))
else:
run_remote_process(
"%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene)
print('Tile generation started')
if options.shutdown:
run(options, 'sudo shutdown 0')
if options.wait and options.local:
while len(threads) > 0:
threads = [t for t in threads if t.is_alive()]
sleep(1)
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
}, "Tile generation controller")
def main():
stats.init_backends({})
parser = ArgumentParser(
description=
'Used to generate the contextual file like the capabilities, the legends, '
'the Apache and MapCache configuration',
prog=sys.argv[0])
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument('--status',
default=False,
action="store_true",
help='Display the SQS queue status and exit')
parser.add_argument('--capabilities',
'--generate-wmts-capabilities',
default=False,
action='store_true',
help='Generate the WMTS Capabilities')
parser.add_argument('--legends',
'--generate-legend-images',
default=False,
action='store_true',
dest='legends',
help='Generate the legend images')
parser.add_argument('--openlayers',
'--generate-openlayers-testpage',
default=False,
action='store_true',
dest='openlayers',
help='Generate openlayers test page')
parser.add_argument('--mapcache',
'--generate-mapcache-config',
default=False,
action='store_true',
dest='mapcache',
help='Generate MapCache configuration file')
parser.add_argument('--apache',
'--generate-apache-config',
default=False,
action='store_true',
dest='apache',
help='Generate Apache configuration file')
parser.add_argument(
'--dump-config',
default=False,
action='store_true',
help='Dump the used config with default values and exit')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(gene)
sys.exit(0)
if options.cache is None:
options.cache = gene.config['generation']['default_cache']
if options.dump_config:
for layer in gene.config['layers'].values():
gene.init_layer(layer, options)
_validate_generate_wmts_capabilities(gene.caches[options.cache], True)
for grid in gene.config['grids'].values():
if 'obj' in grid:
del grid['obj']
print(yaml.dump(gene.config))
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)
def main():
parser = ArgumentParser(
description="Used to calculate the generation cost", prog=sys.argv[0])
add_comon_options(parser, tile_pyramid=False, dimensions=True)
parser.add_argument(
"--cost-algo",
"--calculate-cost-algorithm",
default="area",
dest="cost_algo",
choices=("area", "count"),
help=
"The algorithm use to calculate the cost default base on the 'area' "
"of the generation geometry, can also be 'count', to be base on number of tiles to generate.",
)
options = parser.parse_args()
gene = TileGeneration(options.config,
options,
layer_name=options.layer,
base_config={"cost": {}},
multi_thread=False)
all_size = 0
tile_size = 0
all_tiles = 0
if options.layer:
layer = gene.layers[options.layer]
(all_size, all_time, all_price,
all_tiles) = _calculate_cost(gene, layer, options)
tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024)
else:
all_time = timedelta()
all_price = 0
for layer_name in gene.config["generation"]["default_layers"]:
print("")
print("===== {} =====".format(layer_name))
layer = gene.layers[layer_name]
gene.init_layer(layer, options)
(size, time, price, tiles) = _calculate_cost(gene, layer, options)
tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024)
all_time += time
all_price += price
all_size += size
all_tiles += tiles
print("")
print("===== GLOBAL =====")
print("Total number of tiles: {}".format(all_tiles))
print("Total generation time: {} [d h:mm:ss]".format(
(duration_format(all_time))))
print("Total generation cost: {0:0.2f} [$]".format(all_price))
print("")
print("S3 Storage: {0:0.2f} [$/month]".format(
all_size * gene.config["cost"]["s3"]["storage"] /
(1024.0 * 1024 * 1024)))
print("S3 get: {0:0.2f} [$/month]".format(
(gene.config["cost"]["s3"]["get"] *
gene.config["cost"]["request_per_layers"] / 10000.0 +
gene.config["cost"]["s3"]["download"] *
gene.config["cost"]["request_per_layers"] * tile_size)))
# if 'cloudfront' in gene.config['cost']:
# print('CloudFront: %0.2f [$/month]' % ()
# gene.config['cost']['cloudfront']['get'] *
# gene.config['cost']['request_per_layers'] / 10000.0 +
# gene.config['cost']['cloudfront']['download'] *
# gene.config['cost']['request_per_layers'] * tile_size)
sys.exit(0)
def main():
stats.init_backends({})
parser = ArgumentParser(description="Used to generate the tiles",
prog=sys.argv[0])
add_comon_options(parser, dimensions=True)
parser.add_argument(
"--get-hash",
metavar="TILE",
help="get the empty tiles hash, use the specified TILE z/x/y")
parser.add_argument(
"--get-bbox",
metavar="TILE",
help=
"get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles",
)
parser.add_argument(
"--role",
default="local",
choices=("local", "master", "slave"),
help=
"local/master/slave, master to file the queue and slave to generate the tiles",
)
parser.add_argument("--local-process-number",
default=None,
help="The number of process that we run in parallel")
parser.add_argument("--detach",
default=False,
action="store_true",
help="run detached from the terminal")
parser.add_argument("--daemon",
default=False,
action="store_true",
help="run continuously as a daemon")
parser.add_argument(
"--tiles",
metavar="FILE",
help=
"Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles",
)
options = parser.parse_args()
if options.detach:
detach() # pragma: no cover
gene = TileGeneration(options.config,
options,
multi_thread=options.get_hash is None)
if (options.get_hash is None and options.get_bbox is None
and "authorised_user" in gene.config["generation"]
and gene.config["generation"]["authorised_user"] != getuser()):
sys.exit("not authorised, authorised user is: {}.".format(
gene.config["generation"]["authorised_user"]))
if options.cache is None:
options.cache = gene.config["generation"]["default_cache"]
if options.tiles is not None and options.role not in [
"local", "master"
]: # pragma: no cover
sys.exit("The --tiles option work only with role local or master")
try:
generate = Generate(options, gene)
if options.role == "slave":
generate.gene()
elif options.layer:
generate.gene(gene.layers[options.layer])
elif options.get_bbox: # pragma: no cover
sys.exit("With --get-bbox option you need to specify a layer")
elif options.get_hash: # pragma: no cover
sys.exit("With --get-hash option you need to specify a layer")
elif options.tiles: # pragma: no cover
sys.exit("With --tiles option you need to specify a layer")
else:
for layer in gene.config["generation"].get("default_layers",
gene.layers.keys()):
generate.gene(gene.layers[layer])
finally:
gene.close()
def main():
parser = ArgumentParser(
description="Used to generate the contextual file like the capabilities, the legends, "
"the Apache and MapCache configuration",
prog=sys.argv[0],
)
add_comon_options(parser, tile_pyramid=False, no_geom=False)
parser.add_argument(
"--capabilities",
"--generate-wmts-capabilities",
default=False,
action="store_true",
help="Generate the WMTS Capabilities",
)
parser.add_argument(
"--legends",
"--generate-legend-images",
default=False,
action="store_true",
dest="legends",
help="Generate the legend images",
)
parser.add_argument(
"--openlayers",
"--generate-openlayers-test-page",
default=False,
action="store_true",
dest="openlayers",
help="Generate openlayers test page",
)
parser.add_argument(
"--mapcache",
"--generate-mapcache-config",
default=False,
action="store_true",
dest="mapcache",
help="Generate MapCache configuration file",
)
parser.add_argument(
"--apache",
"--generate-apache-config",
default=False,
action="store_true",
dest="apache",
help="Generate Apache configuration file",
)
parser.add_argument(
"--dump-config", default=False, action="store_true", help="Dump the used config with default values and exit"
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.cache is None:
options.cache = gene.config["generation"]["default_cache"]
if options.dump_config:
for layer in gene.config["layers"].keys():
gene.set_layer(layer, options)
validate_calculate_cost(gene)
_validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
gene.validate_mapcache_config()
gene.validate_apache_config()
_validate_generate_openlayers(gene)
for grid in gene.config["grids"].values():
if "obj" in grid:
del grid["obj"]
print(yaml.dump(gene.config))
sys.exit(0)
if options.legends:
_generate_legend_images(gene)
if options.capabilities:
_generate_wmts_capabilities(gene)
if options.mapcache:
_generate_mapcache_config(gene)
if options.apache:
_generate_apache_config(gene)
if options.openlayers:
_generate_openlayers(gene)