def main(): parser = ArgumentParser( description="Used to copy the tiles from a cache to an other", prog=sys.argv[0]) add_comon_options(parser, near=False, time=False, dimensions=True, cache=False) parser.add_argument("--process", dest="process", metavar="NAME", help="The process name to do") parser.add_argument("source", metavar="SOURCE", help="The source cache") parser.add_argument("dest", metavar="DEST", help="The destination cache") options = parser.parse_args() gene = TileGeneration(options.config, options) if options.layer: # pragma: no cover copy = Copy() copy.copy(options, gene, options.layer, options.source, options.dest, "copy") else: layers = (gene.config["generation"]["default_layers"] if "default_layers" in gene.config["generation"] else gene.config["layers"].keys()) for layer in layers: copy = Copy() copy.copy(options, gene, layer, options.source, options.dest, "copy")
def main(): parser = ArgumentParser( description='Used to copy the tiles from a cache to an other', prog=sys.argv[0]) add_comon_options(parser, near=False, time=False, dimensions=True, cache=False) parser.add_argument('--process', dest='process', metavar="NAME", help='The process name to do') parser.add_argument('source', metavar="SOURCE", help='The source cache') parser.add_argument('dest', metavar="DEST", help='The destination cache') options = parser.parse_args() gene = TileGeneration(options.config, options) if options.layer: # pragma: no cover copy = Copy() copy.copy(options, gene, options.layer, options.source, options.dest, 'copy') else: layers = gene.config['generation']['default_layers'] \ if 'default_layers' in gene.config['generation'] \ else gene.config['layers'].keys() for layer in layers: copy = Copy() copy.copy(options, gene, layer, options.source, options.dest, 'copy')
def main(): parser = ArgumentParser( description='Used to copy the tiles from a cache to an other', prog=sys.argv[0] ) add_comon_options(parser, near=False, time=False, dimensions=True, cache=False) parser.add_argument( '--process', dest='process', metavar="NAME", help='The process name to do' ) parser.add_argument( 'source', metavar="SOURCE", help='The source cache' ) parser.add_argument( 'dest', metavar="DEST", help='The destination cache' ) options = parser.parse_args() gene = TileGeneration(options.config, options) if (options.layer): # pragma: no cover copy = Copy() copy.copy(options, gene, options.layer, options.source, options.dest, 'copy') else: layers = gene.config['generation']['default_layers'] \ if 'default_layers' in gene.config['generation'] \ else gene.config['layers'].keys() for layer in layers: copy = Copy() copy.copy(options, gene, layer, options.source, options.dest, 'copy')
def main(): parser = ArgumentParser( description='Used to calculate the generation cost', prog=sys.argv[0] ) add_comon_options(parser, tile_pyramid=False) parser.add_argument( '--cost-algo', '--calculate-cost-algorithm', default='area', dest='cost_algo', choices=('area', 'count'), help="The algorithm use to calculate the cost default base on the 'area' " "of the generation geometry, can also be 'count', to be base on number of tiles to generate." ) options = parser.parse_args() gene = TileGeneration( options.config, options, layer_name=options.layer, base_config={'cost': {}} ) all_size = 0 tile_size = 0 all_tiles = 0 if (options.layer): (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options) tile_size = gene.layer['cost']['tile_size'] / (1024.0 * 1024) else: all_time = timedelta() all_price = 0 for layer in gene.config['generation']['default_layers']: print("") print("===== %s =====" % layer) gene.set_layer(layer, options) (size, time, price, tiles) = _calculate_cost(gene, options) tile_size += gene.layer['cost']['tile_size'] / (1024.0 * 1024) all_time += time all_price += price all_size += size all_tiles += tiles print("") print("===== GLOBAL =====") print("Total number of tiles: %i" % all_tiles) print('Total generation time: %s [d h:mm:ss]' % (duration_format(all_time))) print('Total generation cost: %0.2f [$]' % all_price) print("") print('S3 Storage: %0.2f [$/month]' % (all_size * gene.config['cost']['s3']['storage'] / (1024.0 * 1024 * 1024))) print('S3 get: %0.2f [$/month]' % ( gene.config['cost']['s3']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 + gene.config['cost']['s3']['download'] * gene.config['cost']['request_per_layers'] * tile_size) ) # if 'cloudfront' in gene.config['cost']: # print('CloudFront: %0.2f [$/month]' % () # gene.config['cost']['cloudfront']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 + # gene.config['cost']['cloudfront']['download'] * gene.config['cost']['request_per_layers'] * tile_size) if 'ec2' in gene.config: print('ESB storage: %0.2f [$/month]' % ( gene.config['cost']['esb']['storage'] * gene.config['cost']['esb_size']) ) sys.exit(0)
def process(): parser = ArgumentParser( description='Used to copy the tiles from a cache to an other', prog='./buildout/bin/generate_copy' ) add_comon_options(parser, near=False, time=False, dimensions=True) parser.add_argument( 'process', metavar="PROCESS", help='The process name to do' ) options = parser.parse_args() gene = TileGeneration(options.config, options) if (options.layer): # pragma: no cover copy = Copy() copy.copy(options, gene, options.layer, options.cache, options.cache, 'process') else: for layer in gene.config['generation']['default_layers']: copy = Copy() copy.copy(options, gene, layer, options.cache, options.cache, 'process')
def process(): parser = ArgumentParser( description='Used to copy the tiles from a cache to an other', prog='./buildout/bin/generate_copy') add_comon_options(parser, near=False, time=False, dimensions=True) parser.add_argument('process', metavar="PROCESS", help='The process name to do') options = parser.parse_args() gene = TileGeneration(options.config, options) if (options.layer): # pragma: no cover copy = Copy() copy.copy(options, gene, options.layer, options.cache, options.cache, 'process') else: for layer in gene.config['generation']['default_layers']: copy = Copy() copy.copy(options, gene, layer, options.cache, options.cache, 'process')
def process(): parser = ArgumentParser( description='Used to copy the tiles from a cache to an other', prog=sys.argv[0] ) add_comon_options(parser, near=False, time=False, dimensions=True) parser.add_argument( 'process', metavar="PROCESS", help='The process name to do' ) options = parser.parse_args() gene = TileGeneration(options.config, options) copy = Copy() if (options.layer): # pragma: no cover copy.copy(options, gene, options.layer, options.cache, options.cache, 'process') else: layers_name = gene.config['generation']['default_layers'] \ if 'default_layers' in gene.config.get('generation', {}) \ else gene.layers.keys() for layer in layers_name: copy.copy(options, gene, layer, options.cache, options.cache, 'process')
def process(): parser = ArgumentParser( description='Used to copy the tiles from a cache to an other', prog=sys.argv[0]) add_comon_options(parser, near=False, time=False, dimensions=True) parser.add_argument('process', metavar="PROCESS", help='The process name to do') options = parser.parse_args() gene = TileGeneration(options.config, options) copy = Copy() if options.layer: # pragma: no cover copy.copy(options, gene, options.layer, options.cache, options.cache, 'process') else: layers_name = gene.config['generation']['default_layers'] \ if 'default_layers' in gene.config.get('generation', {}) \ else gene.layers.keys() for layer in layers_name: copy.copy(options, gene, layer, options.cache, options.cache, 'process')
def process(): parser = ArgumentParser( description="Used to copy the tiles from a cache to an other", prog=sys.argv[0]) add_comon_options(parser, near=False, time=False, dimensions=True) parser.add_argument("process", metavar="PROCESS", help="The process name to do") options = parser.parse_args() gene = TileGeneration(options.config, options, multi_thread=False) copy = Copy() if options.layer: # pragma: no cover copy.copy(options, gene, options.layer, options.cache, options.cache, "process") else: layers_name = (gene.config["generation"]["default_layers"] if "default_layers" in gene.config.get( "generation", {}) else gene.layers.keys()) for layer in layers_name: copy.copy(options, gene, layer, options.cache, options.cache, "process")
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog='./buildout/bin/generate_amazon') add_comon_options(parser) parser.add_argument('--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file') parser.add_argument('--status', default=False, action="store_true", help='display the SQS queue status and exit') parser.add_argument('--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation') parser.add_argument('--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code') parser.add_argument('--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database') parser.add_argument('--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling') parser.add_argument('--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation') parser.add_argument('--host', default=None, help='The host used to generate tiles') parser.add_argument('--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.') options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print "EC2 not configured" sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if options.geodata and 'geodata_folder' in gene.config['ec2']: print "==== Sync geodata ====" ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code: print "==== Sync and build code ====" error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True) if error: exit(1) # pragma: no cover cmd = [ 'rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run_remote(cmd, host, project_dir, gene) if 'apache_content' in gene.config[ 'ec2'] and 'apache_config' in gene.config['ec2']: run_remote( 'echo %s > %s' % (gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config']), host, project_dir, gene) run_remote('sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene)) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce(lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta()) / len(times)**2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print '==== Time results ====' print 'A tile is generated in: %0.3f [ms]' % mean_time_ms print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb) print '''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb) if options.shutdown: # pragma: no cover run_remote('sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue: # pragma: no cover print "==== Till queue ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master']) project_dir = gene.config['ec2']['code_folder'] run_remote( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene) if options.tiles_gen: # pragma: no cover print "==== Generate tiles ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'slave']) arguments.append("--daemonize") project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene)) if options.shutdown: for p in processes: p.communicate() # wait process end else: print 'Tile generation started in background' if options.shutdown: run_remote('sudo shutdown 0') if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller")
def main(): stats.init_backends({}) parser = ArgumentParser( description="Used to generate the contextual file like the capabilities, the legends, " "the Apache and MapCache configuration", prog=sys.argv[0], ) add_comon_options(parser, tile_pyramid=False, no_geom=False) parser.add_argument( "--status", default=False, action="store_true", help="Display the SQS queue status and exit" ) parser.add_argument( "--capabilities", "--generate-wmts-capabilities", default=False, action="store_true", help="Generate the WMTS Capabilities", ) parser.add_argument( "--legends", "--generate-legend-images", default=False, action="store_true", dest="legends", help="Generate the legend images", ) parser.add_argument( "--openlayers", "--generate-openlayers-testpage", default=False, action="store_true", dest="openlayers", help="Generate openlayers test page", ) parser.add_argument( "--mapcache", "--generate-mapcache-config", default=False, action="store_true", dest="mapcache", help="Generate MapCache configuration file", ) parser.add_argument( "--mapcache-version", default="1.4", choices=("1.4", "1.6"), help="The used version of MapCache" ) parser.add_argument( "--apache", "--generate-apache-config", default=False, action="store_true", dest="apache", help="Generate Apache configuration file", ) parser.add_argument( "--dump-config", default=False, action="store_true", help="Dump the used config with default values and exit", ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(gene) sys.exit(0) if options.cache is None: options.cache = gene.config["generation"]["default_cache"] if options.dump_config: for layer in gene.config["layers"].values(): gene.init_layer(layer, options) _validate_generate_wmts_capabilities(gene.caches[options.cache], True) for grid in gene.config["grids"].values(): if "obj" in grid: del grid["obj"] print(yaml.dump(gene.config)) sys.exit(0) if options.legends: _generate_legend_images(gene) if options.capabilities: _generate_wmts_capabilities(gene) if options.mapcache: _generate_mapcache_config(gene, options.mapcache_version) if options.apache: _generate_apache_config(gene) if options.openlayers: _generate_openlayers(gene)
def main(): parser = ArgumentParser(description='Used to generate the tiles', prog='./buildout/bin/generate_tiles') add_comon_options(parser, dimensions=True) parser.add_argument( '--get-hash', metavar="TILE", help='get the empty tiles hash, use the specified TILE z/x/y') parser.add_argument( '--get-bbox', metavar="TILE", help= 'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument( '--role', default='local', choices=('local', 'master', 'slave'), help='local/master/slave, master to file the queue and ' 'slave to generate the tiles') parser.add_argument('--daemonize', default=False, action="store_true", help='run as a daemon') parser.add_argument( '--tiles', metavar="FILE", help= 'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument('--generated-tiles-file', metavar="FILE", help='Store the tiles in a file (unrecommended)') options = parser.parse_args() if options.daemonize: daemonize() # pragma: no cover gene = TileGeneration(options.config, options) if options.get_hash is None and options.get_bbox is None and \ 'authorised_user' in gene.config['generation'] and \ gene.config['generation']['authorised_user'] != getuser(): exit('not authorised, authorised user is: %s.' % gene.config['generation']['authorised_user']) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.tiles is not None and options.role not in [ 'local', 'master' ]: # pragma: no cover exit("The --tiles option work only with role local or master") try: if (options.layer): generate = Generate() generate.gene(options, gene, options.layer) elif options.get_bbox: # pragma: no cover exit("With --get-bbox option we needs to specify a layer") elif options.get_hash: # pragma: no cover exit("With --get-hash option we needs to specify a layer") elif options.tiles: # pragma: no cover exit("With --tiles option we needs to specify a layer") else: for layer in gene.config['generation']['default_layers']: generate = Generate() generate.gene(options, gene, layer) finally: if gene.error_file is not None: gene.error_file.close()
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog='./buildout/bin/generate_amazon' ) add_comon_options(parser) parser.add_argument( '--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file' ) parser.add_argument( '--status', default=False, action="store_true", help='display the SQS queue status and exit' ) parser.add_argument( '--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation' ) parser.add_argument( '--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code' ) parser.add_argument( '--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database' ) parser.add_argument( '--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling' ) parser.add_argument( '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation' ) parser.add_argument( '--host', default=None, help='The host used to generate tiles' ) parser.add_argument( '--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.' ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print "EC2 not configured" sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if options.geodata and 'geodata_folder' in gene.config['ec2']: print "==== Sync geodata ====" ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code: print "==== Sync and build code ====" error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True) if error: exit(1) # pragma: no cover cmd = ['rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run_remote(cmd, host, project_dir, gene) if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']: run_remote( 'echo %s > %s' % ( gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config'] ), host, project_dir, gene ) run_remote('sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene ) ) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce( lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta() ) / len(times) ** 2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print '==== Time results ====' print 'A tile is generated in: %0.3f [ms]' % mean_time_ms print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb) print '''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb) if options.shutdown: # pragma: no cover run_remote('sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue: # pragma: no cover print "==== Till queue ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master']) project_dir = gene.config['ec2']['code_folder'] run_remote( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene ) if options.tiles_gen: # pragma: no cover print "==== Generate tiles ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'slave']) arguments.append("--daemonize") project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene) ) if options.shutdown: for p in processes: p.communicate() # wait process end else: print 'Tile generation started in background' if options.shutdown: run_remote('sudo shutdown 0') if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller")
def main(): parser = ArgumentParser( description='Used to generate the contextual file like the capabilities, the legends, ' 'the Apache and MapCache configuration', prog='./buildout/bin/generate_controller' ) add_comon_options(parser, tile_pyramid=False, no_geom=False) parser.add_argument( '--capabilities', '--generate_wmts_capabilities', default=False, action="store_true", help='Generate the WMTS Capabilities' ) parser.add_argument( '--legends', '--generate_legend_images', default=False, action="store_true", dest='legends', help='Generate the legend images' ) parser.add_argument( '--openlayers', '--generate-openlayers-test-page', default=False, action="store_true", dest='openlayers', help='Generate openlayers test page' ) parser.add_argument( '--mapcache', '--generate-mapcache-config', default=False, action="store_true", dest='mapcache', help='Generate MapCache configuration file' ) parser.add_argument( '--apache', '--generate-apache-config', default=False, action="store_true", dest='apache', help='Generate Apache configuration file' ) parser.add_argument( '--dump-config', default=False, action="store_true", help='Dump the used config with default values and exit' ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.dump_config: for layer in gene.config['layers'].keys(): gene.set_layer(layer, options) validate_calculate_cost(gene) _validate_generate_wmts_capabilities(gene, gene.caches[options.cache]) gene.validate_mapcache_config() gene.validate_apache_config() _validate_generate_openlayers(gene) for grid in gene.config['grids'].values(): if 'obj' in grid: del grid['obj'] print yaml.dump(gene.config) sys.exit(0) if options.legends: _generate_legend_images(gene) if options.capabilities: _generate_wmts_capabilities(gene) if options.mapcache: _generate_mapcache_config(gene) if options.apache: _generate_apache_config(gene) if options.openlayers: _generate_openlayers(gene)
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog=sys.argv[0] ) add_comon_options(parser) parser.add_argument( '--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file' ) parser.add_argument( '--status', default=False, action="store_true", help='display the SQS queue status and exit' ) parser.add_argument( '--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation' ) parser.add_argument( '--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code' ) parser.add_argument( '--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database' ) parser.add_argument( '--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling' ) parser.add_argument( '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation' ) parser.add_argument( '--host', default=None, help='The host used to generate tiles' ) parser.add_argument( '--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.' ) parser.add_argument( '--wait', default=False, action="store_true", help='Wait that all the tasks will finish.' ) parser.add_argument( '--local', default=False, action="store_true", help='Run the generation locally' ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print("EC2 not configured") sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']: print("==== Sync geodata ====") ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code and not options.local: print("==== Sync and build code ====") error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True) if error: exit(1) # pragma: no cover cmd = ['rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run(options, cmd % environ, host, project_dir, gene) if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']: run( options, 'echo %s > %s' % ( gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config'] ), host, project_dir, gene ) run(options, 'sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database and not options.local: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata and not options.local: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = None if options.local else gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( "%sgenerate_tiles %s" % ( _get_path(), ' '.join([str(a) for a in arguments]) ), host, project_dir, gene ) ) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) if PY3: results = [r.decode('utf-8') for r in results] results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce( lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta() ) / len(times) ** 2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print('==== Time results ====') print('A tile is generated in: %0.3f [ms]' % mean_time_ms) print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)) print('''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)) if options.shutdown: # pragma: no cover run(options, 'sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue and not options.local: # pragma: no cover print("==== Till queue ====") # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master', '--quiet']) project_dir = gene.config['ec2']['code_folder'] run_remote_process( options, "%sgenerate_tiles %s" % ( _get_path(), ' '.join([str(a) for a in arguments]) ), host, project_dir, gene ) sleep(5) attributes = gene.get_sqs_queue().get_attributes() print( "\rTiles to generate: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], ) ) if options.tiles_gen: # pragma: no cover print("==== Generate tiles ====") if options.wait and not options.local: print("") class Status(Thread): def run(self): # pragma: no cover while True: attributes = gene.get_sqs_queue().get_attributes() print( "\rTiles to generate/generating: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], ) ) sleep(1) status_thread = Status() status_thread.setDaemon(True) status_thread.start() arguments = _get_arguments(options) arguments.extend(['--quiet']) if not options.local: arguments.extend(['--role', 'slave']) project_dir = None if options.local else gene.config['ec2']['code_folder'] threads = [] for i in range(gene.config['ec2']['number_process']): if options.local: threads.append(run_local_process( "%sgenerate_tiles --local-process-number %i %s" % ( _get_path(), i, ' '.join([str(a) for a in arguments]) ) )) else: run_remote_process( "%sgenerate_tiles %s" % ( _get_path(), ' '.join([str(a) for a in arguments]) ), host, project_dir, gene ) print('Tile generation started') if options.shutdown: run(options, 'sudo shutdown 0') if options.wait and options.local: while len(threads) > 0: threads = [t for t in threads if t.is_alive()] sleep(1) if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller" )
def main(): parser = ArgumentParser( description= 'Used to generate the contextual file like the capabilities, the legends, ' 'the Apache and MapCache configuration', prog='./buildout/bin/generate_controller') add_comon_options(parser, tile_pyramid=False, no_geom=False) parser.add_argument('--capabilities', '--generate_wmts_capabilities', default=False, action="store_true", help='Generate the WMTS Capabilities') parser.add_argument('--legends', '--generate_legend_images', default=False, action="store_true", dest='legends', help='Generate the legend images') parser.add_argument('--openlayers', '--generate-openlayers-test-page', default=False, action="store_true", dest='openlayers', help='Generate openlayers test page') parser.add_argument('--mapcache', '--generate-mapcache-config', default=False, action="store_true", dest='mapcache', help='Generate MapCache configuration file') parser.add_argument('--apache', '--generate-apache-config', default=False, action="store_true", dest='apache', help='Generate Apache configuration file') parser.add_argument( '--dump-config', default=False, action="store_true", help='Dump the used config with default values and exit') options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.dump_config: for layer in gene.config['layers'].keys(): gene.set_layer(layer, options) validate_calculate_cost(gene) _validate_generate_wmts_capabilities(gene, gene.caches[options.cache]) gene.validate_mapcache_config() gene.validate_apache_config() _validate_generate_openlayers(gene) for grid in gene.config['grids'].values(): if 'obj' in grid: del grid['obj'] print yaml.dump(gene.config) sys.exit(0) if options.legends: _generate_legend_images(gene) if options.capabilities: _generate_wmts_capabilities(gene) if options.mapcache: _generate_mapcache_config(gene) if options.apache: _generate_apache_config(gene) if options.openlayers: _generate_openlayers(gene)
def main(): parser = ArgumentParser(description='Used to generate the tiles', prog=sys.argv[0]) add_comon_options(parser, dimensions=True) parser.add_argument( '--get-hash', metavar="TILE", help='get the empty tiles hash, use the specified TILE z/x/y' ) parser.add_argument( '--get-bbox', metavar="TILE", help='get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument( '--role', default='local', choices=('local', 'master', 'slave'), help='local/master/slave, master to file the queue and ' 'slave to generate the tiles' ) parser.add_argument( "--local-process-number", default=None, help="The number of process that we run in parallel" ) parser.add_argument( '--daemonize', default=False, action="store_true", help='run as a daemon' ) parser.add_argument( '--tiles', metavar="FILE", help='Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument( '--generated-tiles-file', metavar="FILE", help='Store the tiles in a file (unrecommended)' ) options = parser.parse_args() if options.daemonize: daemonize() # pragma: no cover gene = TileGeneration(options.config, options) if options.get_hash is None and options.get_bbox is None and \ 'authorised_user' in gene.config['generation'] and \ gene.config['generation']['authorised_user'] != getuser(): exit('not authorised, authorised user is: %s.' % gene.config['generation']['authorised_user']) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.tiles is not None and options.role not in ['local', 'master']: # pragma: no cover exit("The --tiles option work only with role local or master") try: if (options.layer): generate = Generate() generate.gene(options, gene, options.layer) elif options.get_bbox: # pragma: no cover exit("With --get-bbox option we needs to specify a layer") elif options.get_hash: # pragma: no cover exit("With --get-hash option we needs to specify a layer") elif options.tiles: # pragma: no cover exit("With --tiles option we needs to specify a layer") else: for layer in gene.config['generation'].get('default_layers', gene.layers.keys()): generate = Generate() generate.gene(options, gene, layer) finally: if gene.error_file is not None: gene.error_file.close()
def main(): parser = ArgumentParser(description='Used to generate the tiles', prog=sys.argv[0]) add_comon_options(parser, dimensions=True) parser.add_argument( '--get-hash', metavar="TILE", help='get the empty tiles hash, use the specified TILE z/x/y') parser.add_argument( '--get-bbox', metavar="TILE", help= 'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument( '--role', default='local', choices=('local', 'master', 'slave'), help='local/master/slave, master to file the queue and ' 'slave to generate the tiles') parser.add_argument("--local-process-number", default=None, help="The number of process that we run in parallel") parser.add_argument('--detach', default=False, action="store_true", help='run detached from the terminal') parser.add_argument('--daemon', default=False, action="store_true", help='run continuously as a daemon') parser.add_argument( '--tiles', metavar="FILE", help= 'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument('--generated-tiles-file', metavar="FILE", help='Store the tiles in a file (unrecommended)') options = parser.parse_args() if options.detach: detach() # pragma: no cover gene = TileGeneration(options.config, options) if options.get_hash is None and options.get_bbox is None and \ 'authorised_user' in gene.config['generation'] and \ gene.config['generation']['authorised_user'] != getuser(): exit('not authorised, authorised user is: {}.'.format( gene.config['generation']['authorised_user'])) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.tiles is not None and options.role not in [ 'local', 'master' ]: # pragma: no cover exit("The --tiles option work only with role local or master") try: if options.role == 'slave': generate = Generate() generate.gene(options, gene) elif (options.layer): generate = Generate() generate.gene(options, gene, options.layer) elif options.get_bbox: # pragma: no cover exit("With --get-bbox option we needs to specify a layer") elif options.get_hash: # pragma: no cover exit("With --get-hash option we needs to specify a layer") elif options.tiles: # pragma: no cover exit("With --tiles option we needs to specify a layer") else: for layer in gene.config['generation'].get('default_layers', gene.layers.keys()): generate = Generate() generate.gene(options, gene, layer) finally: if gene.error_file is not None: gene.error_file.close()
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog=sys.argv[0]) add_comon_options(parser) parser.add_argument('--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file') parser.add_argument('--status', default=False, action="store_true", help='display the SQS queue status and exit') parser.add_argument('--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation') parser.add_argument('--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code') parser.add_argument('--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database') parser.add_argument('--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling') parser.add_argument('--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation') parser.add_argument('--host', default=None, help='The host used to generate tiles') parser.add_argument('--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.') parser.add_argument('--wait', default=False, action="store_true", help='Wait that all the tasks will finish.') parser.add_argument('--local', default=False, action="store_true", help='Run the generation locally') options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print("EC2 not configured") sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if not options.local and options.geodata and 'geodata_folder' in gene.config[ 'ec2']: # pragma: no cover print("==== Sync geodata ====") ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code and not options.local: print("==== Sync and build code ====") cmd = [ 'rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run(options, cmd % environ, host, project_dir, gene) if 'apache_content' in gene.config[ 'ec2'] and 'apache_config' in gene.config['ec2']: run( options, 'echo %s > %s' % (gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config']), host, project_dir, gene) run(options, 'sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database and not options.local: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata and not options.local: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = None if options.local else gene.config['ec2'][ 'code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( "%sgenerate_tiles %s" % (_get_path(), ' '.join([str(a) for a in arguments])), host, project_dir, gene)) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) if PY3: results = [r.decode('utf-8') for r in results] results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce(lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta()) / len(times)**2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print('==== Time results ====') print('A tile is generated in: %0.3f [ms]' % mean_time_ms) print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)) print('''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)) if options.shutdown: # pragma: no cover run(options, 'sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue and not options.local: # pragma: no cover print("==== Till queue ====") # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master', '--quiet']) project_dir = gene.config['ec2']['code_folder'] run_remote_process( options, "%sgenerate_tiles %s" % (_get_path(), ' '.join([str(a) for a in arguments])), host, project_dir, gene) sleep(5) attributes = gene.get_sqs_queue().get_attributes() print("\rTiles to generate: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], )) if options.tiles_gen: # pragma: no cover print("==== Generate tiles ====") if options.wait and not options.local: print("") class Status(Thread): def run(self): # pragma: no cover while True: attributes = gene.get_sqs_queue().get_attributes() print("\rTiles to generate/generating: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], )) sleep(1) status_thread = Status() status_thread.setDaemon(True) status_thread.start() arguments = _get_arguments(options) arguments.extend(['--quiet']) if not options.local: arguments.extend(['--role', 'slave']) project_dir = None if options.local else gene.config['ec2'][ 'code_folder'] threads = [] for i in range(gene.config['ec2']['number_process']): if options.local: threads.append( run_local_process( "%sgenerate_tiles --local-process-number %i %s" % (_get_path(), i, ' '.join([str(a) for a in arguments])))) else: run_remote_process( "%sgenerate_tiles %s" % (_get_path(), ' '.join([str(a) for a in arguments])), host, project_dir, gene) print('Tile generation started') if options.shutdown: run(options, 'sudo shutdown 0') if options.wait and options.local: while len(threads) > 0: threads = [t for t in threads if t.is_alive()] sleep(1) if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller")
def main(): stats.init_backends({}) parser = ArgumentParser( description= 'Used to generate the contextual file like the capabilities, the legends, ' 'the Apache and MapCache configuration', prog=sys.argv[0]) add_comon_options(parser, tile_pyramid=False, no_geom=False) parser.add_argument('--status', default=False, action="store_true", help='Display the SQS queue status and exit') parser.add_argument('--capabilities', '--generate-wmts-capabilities', default=False, action='store_true', help='Generate the WMTS Capabilities') parser.add_argument('--legends', '--generate-legend-images', default=False, action='store_true', dest='legends', help='Generate the legend images') parser.add_argument('--openlayers', '--generate-openlayers-testpage', default=False, action='store_true', dest='openlayers', help='Generate openlayers test page') parser.add_argument('--mapcache', '--generate-mapcache-config', default=False, action='store_true', dest='mapcache', help='Generate MapCache configuration file') parser.add_argument('--apache', '--generate-apache-config', default=False, action='store_true', dest='apache', help='Generate Apache configuration file') parser.add_argument( '--dump-config', default=False, action='store_true', help='Dump the used config with default values and exit') options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(gene) sys.exit(0) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.dump_config: for layer in gene.config['layers'].values(): gene.init_layer(layer, options) _validate_generate_wmts_capabilities(gene.caches[options.cache], True) for grid in gene.config['grids'].values(): if 'obj' in grid: del grid['obj'] print(yaml.dump(gene.config)) sys.exit(0) if options.legends: _generate_legend_images(gene) if options.capabilities: _generate_wmts_capabilities(gene) if options.mapcache: _generate_mapcache_config(gene) if options.apache: _generate_apache_config(gene) if options.openlayers: _generate_openlayers(gene)
def main(): parser = ArgumentParser( description="Used to calculate the generation cost", prog=sys.argv[0]) add_comon_options(parser, tile_pyramid=False, dimensions=True) parser.add_argument( "--cost-algo", "--calculate-cost-algorithm", default="area", dest="cost_algo", choices=("area", "count"), help= "The algorithm use to calculate the cost default base on the 'area' " "of the generation geometry, can also be 'count', to be base on number of tiles to generate.", ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer, base_config={"cost": {}}, multi_thread=False) all_size = 0 tile_size = 0 all_tiles = 0 if options.layer: layer = gene.layers[options.layer] (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, layer, options) tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024) else: all_time = timedelta() all_price = 0 for layer_name in gene.config["generation"]["default_layers"]: print("") print("===== {} =====".format(layer_name)) layer = gene.layers[layer_name] gene.init_layer(layer, options) (size, time, price, tiles) = _calculate_cost(gene, layer, options) tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024) all_time += time all_price += price all_size += size all_tiles += tiles print("") print("===== GLOBAL =====") print("Total number of tiles: {}".format(all_tiles)) print("Total generation time: {} [d h:mm:ss]".format( (duration_format(all_time)))) print("Total generation cost: {0:0.2f} [$]".format(all_price)) print("") print("S3 Storage: {0:0.2f} [$/month]".format( all_size * gene.config["cost"]["s3"]["storage"] / (1024.0 * 1024 * 1024))) print("S3 get: {0:0.2f} [$/month]".format( (gene.config["cost"]["s3"]["get"] * gene.config["cost"]["request_per_layers"] / 10000.0 + gene.config["cost"]["s3"]["download"] * gene.config["cost"]["request_per_layers"] * tile_size))) # if 'cloudfront' in gene.config['cost']: # print('CloudFront: %0.2f [$/month]' % () # gene.config['cost']['cloudfront']['get'] * # gene.config['cost']['request_per_layers'] / 10000.0 + # gene.config['cost']['cloudfront']['download'] * # gene.config['cost']['request_per_layers'] * tile_size) sys.exit(0)
def main(): stats.init_backends({}) parser = ArgumentParser(description="Used to generate the tiles", prog=sys.argv[0]) add_comon_options(parser, dimensions=True) parser.add_argument( "--get-hash", metavar="TILE", help="get the empty tiles hash, use the specified TILE z/x/y") parser.add_argument( "--get-bbox", metavar="TILE", help= "get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles", ) parser.add_argument( "--role", default="local", choices=("local", "master", "slave"), help= "local/master/slave, master to file the queue and slave to generate the tiles", ) parser.add_argument("--local-process-number", default=None, help="The number of process that we run in parallel") parser.add_argument("--detach", default=False, action="store_true", help="run detached from the terminal") parser.add_argument("--daemon", default=False, action="store_true", help="run continuously as a daemon") parser.add_argument( "--tiles", metavar="FILE", help= "Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles", ) options = parser.parse_args() if options.detach: detach() # pragma: no cover gene = TileGeneration(options.config, options, multi_thread=options.get_hash is None) if (options.get_hash is None and options.get_bbox is None and "authorised_user" in gene.config["generation"] and gene.config["generation"]["authorised_user"] != getuser()): sys.exit("not authorised, authorised user is: {}.".format( gene.config["generation"]["authorised_user"])) if options.cache is None: options.cache = gene.config["generation"]["default_cache"] if options.tiles is not None and options.role not in [ "local", "master" ]: # pragma: no cover sys.exit("The --tiles option work only with role local or master") try: generate = Generate(options, gene) if options.role == "slave": generate.gene() elif options.layer: generate.gene(gene.layers[options.layer]) elif options.get_bbox: # pragma: no cover sys.exit("With --get-bbox option you need to specify a layer") elif options.get_hash: # pragma: no cover sys.exit("With --get-hash option you need to specify a layer") elif options.tiles: # pragma: no cover sys.exit("With --tiles option you need to specify a layer") else: for layer in gene.config["generation"].get("default_layers", gene.layers.keys()): generate.gene(gene.layers[layer]) finally: gene.close()
def main(): parser = ArgumentParser( description="Used to generate the contextual file like the capabilities, the legends, " "the Apache and MapCache configuration", prog=sys.argv[0], ) add_comon_options(parser, tile_pyramid=False, no_geom=False) parser.add_argument( "--capabilities", "--generate-wmts-capabilities", default=False, action="store_true", help="Generate the WMTS Capabilities", ) parser.add_argument( "--legends", "--generate-legend-images", default=False, action="store_true", dest="legends", help="Generate the legend images", ) parser.add_argument( "--openlayers", "--generate-openlayers-test-page", default=False, action="store_true", dest="openlayers", help="Generate openlayers test page", ) parser.add_argument( "--mapcache", "--generate-mapcache-config", default=False, action="store_true", dest="mapcache", help="Generate MapCache configuration file", ) parser.add_argument( "--apache", "--generate-apache-config", default=False, action="store_true", dest="apache", help="Generate Apache configuration file", ) parser.add_argument( "--dump-config", default=False, action="store_true", help="Dump the used config with default values and exit" ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.cache is None: options.cache = gene.config["generation"]["default_cache"] if options.dump_config: for layer in gene.config["layers"].keys(): gene.set_layer(layer, options) validate_calculate_cost(gene) _validate_generate_wmts_capabilities(gene, gene.caches[options.cache]) gene.validate_mapcache_config() gene.validate_apache_config() _validate_generate_openlayers(gene) for grid in gene.config["grids"].values(): if "obj" in grid: del grid["obj"] print(yaml.dump(gene.config)) sys.exit(0) if options.legends: _generate_legend_images(gene) if options.capabilities: _generate_wmts_capabilities(gene) if options.mapcache: _generate_mapcache_config(gene) if options.apache: _generate_apache_config(gene) if options.openlayers: _generate_openlayers(gene)