def process() -> None: """Copy the tiles from a cache to an other.""" try: parser = ArgumentParser( description="Used to copy the tiles from a cache to an other", prog=sys.argv[0]) add_common_options(parser, near=False, time=False, dimensions=True) parser.add_argument("process", metavar="PROCESS", help="The process name to do") options = parser.parse_args() gene = TileGeneration(options.config, options, multi_thread=False) copy = Copy() if options.layer: copy.copy(options, gene, options.layer, options.cache, options.cache, "process") else: assert gene.config_file config = gene.get_config(gene.config_file) layers_name = (config.config["generation"]["default_layers"] if "default_layers" in config.config.get( "generation", {}) else config.config["layers"].keys()) for layer in layers_name: copy.copy(options, gene, layer, options.cache, options.cache, "process") except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") sys.exit(1)
def main(): parser = ArgumentParser( description='Used to calculate the generation cost', prog=sys.argv[0] ) add_comon_options(parser, tile_pyramid=False) parser.add_argument( '--cost-algo', '--calculate-cost-algorithm', default='area', dest='cost_algo', choices=('area', 'count'), help="The algorithm use to calculate the cost default base on the 'area' " "of the generation geometry, can also be 'count', to be base on number of tiles to generate." ) options = parser.parse_args() gene = TileGeneration( options.config, options, layer_name=options.layer, base_config={'cost': {}} ) all_size = 0 tile_size = 0 all_tiles = 0 if (options.layer): (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options) tile_size = gene.layer['cost']['tile_size'] / (1024.0 * 1024) else: all_time = timedelta() all_price = 0 for layer in gene.config['generation']['default_layers']: print("") print("===== %s =====" % layer) gene.set_layer(layer, options) (size, time, price, tiles) = _calculate_cost(gene, options) tile_size += gene.layer['cost']['tile_size'] / (1024.0 * 1024) all_time += time all_price += price all_size += size all_tiles += tiles print("") print("===== GLOBAL =====") print("Total number of tiles: %i" % all_tiles) print('Total generation time: %s [d h:mm:ss]' % (duration_format(all_time))) print('Total generation cost: %0.2f [$]' % all_price) print("") print('S3 Storage: %0.2f [$/month]' % (all_size * gene.config['cost']['s3']['storage'] / (1024.0 * 1024 * 1024))) print('S3 get: %0.2f [$/month]' % ( gene.config['cost']['s3']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 + gene.config['cost']['s3']['download'] * gene.config['cost']['request_per_layers'] * tile_size) ) # if 'cloudfront' in gene.config['cost']: # print('CloudFront: %0.2f [$/month]' % () # gene.config['cost']['cloudfront']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 + # gene.config['cost']['cloudfront']['download'] * gene.config['cost']['request_per_layers'] * tile_size) if 'ec2' in gene.config: print('ESB storage: %0.2f [$/month]' % ( gene.config['cost']['esb']['storage'] * gene.config['cost']['esb_size']) ) sys.exit(0)
def _fill_legend( gene: TileGeneration, cache: tilecloud_chain.configuration.Cache, server: Optional[tilecloud_chain.configuration.Server], base_urls: List[str], ) -> None: assert gene.config_file config = gene.get_config(gene.config_file) for layer_name, layer in config.config["layers"].items(): previous_legend: Optional[tilecloud_chain.Legend] = None previous_resolution = None if "legend_mime" in layer and "legend_extension" in layer and layer_name not in gene.layer_legends: gene.layer_legends[layer_name] = [] legends = gene.layer_legends[layer_name] for zoom, resolution in enumerate( config.config["grids"][layer["grid"]]["resolutions"]): path = "/".join([ "1.0.0", layer_name, layer["wmts_style"], f"legend{zoom}.{layer['legend_extension']}", ]) img = _get(path, cache) if img is not None: new_legend: tilecloud_chain.Legend = { "mime_type": layer["legend_mime"], "href": os.path.join( base_urls[0], server.get("static_path", "static") + "/" if server else "", path), } legends.append(new_legend) if previous_legend is not None: assert previous_resolution is not None middle_res = exp( (log(previous_resolution) + log(resolution)) / 2) previous_legend[ # pylint: disable=unsupported-assignment-operation "min_resolution"] = middle_res new_legend["max_resolution"] = middle_res try: pil_img = Image.open(BytesIO(img)) new_legend["width"] = pil_img.size[0] new_legend["height"] = pil_img.size[1] except Exception: # pragma: nocover logger.warning( "Unable to read legend image '%s', with '%s'", path, repr(img), exc_info=True, ) previous_legend = new_legend previous_resolution = resolution
def init_tilegeneration(config_file: Optional[str]) -> None: """Initialize the tile generation.""" global tilegeneration # pylint: disable=global-statement if tilegeneration is None: logger.info("Config file: '%s'", config_file) log_level = os.environ.get("TILE_SERVER_LOGLEVEL") tilegeneration = TileGeneration( config_file, collections.namedtuple( # type: ignore "Options", [ "verbose", "debug", "quiet", "bbox", "zoom", "test", "near", "time", "geom", "ignore_error" ], )( log_level == "verbose", # type: ignore log_level == "debug", log_level == "quiet", None, None, None, None, None, True, False, ), configure_logging=False, multi_thread=False, maxconsecutive_errors=False, )
def main(): parser = ArgumentParser( description='Used to copy the tiles from a cache to an other', prog=sys.argv[0]) add_comon_options(parser, near=False, time=False, dimensions=True, cache=False) parser.add_argument('--process', dest='process', metavar="NAME", help='The process name to do') parser.add_argument('source', metavar="SOURCE", help='The source cache') parser.add_argument('dest', metavar="DEST", help='The destination cache') options = parser.parse_args() gene = TileGeneration(options.config, options) if options.layer: # pragma: no cover copy = Copy() copy.copy(options, gene, options.layer, options.source, options.dest, 'copy') else: layers = gene.config['generation']['default_layers'] \ if 'default_layers' in gene.config['generation'] \ else gene.config['layers'].keys() for layer in layers: copy = Copy() copy.copy(options, gene, layer, options.source, options.dest, 'copy')
def main(): parser = ArgumentParser( description="Used to copy the tiles from a cache to an other", prog=sys.argv[0]) add_comon_options(parser, near=False, time=False, dimensions=True, cache=False) parser.add_argument("--process", dest="process", metavar="NAME", help="The process name to do") parser.add_argument("source", metavar="SOURCE", help="The source cache") parser.add_argument("dest", metavar="DEST", help="The destination cache") options = parser.parse_args() gene = TileGeneration(options.config, options) if options.layer: # pragma: no cover copy = Copy() copy.copy(options, gene, options.layer, options.source, options.dest, "copy") else: layers = (gene.config["generation"]["default_layers"] if "default_layers" in gene.config["generation"] else gene.config["layers"].keys()) for layer in layers: copy = Copy() copy.copy(options, gene, layer, options.source, options.dest, "copy")
def main() -> None: """Copy the tiles from a cache to an other.""" try: parser = ArgumentParser( description="Used to copy the tiles from a cache to an other", prog=sys.argv[0]) add_common_options(parser, near=False, time=False, dimensions=True, cache=False) parser.add_argument("--process", dest="process", metavar="NAME", help="The process name to do") parser.add_argument("source", metavar="SOURCE", help="The source cache") parser.add_argument("dest", metavar="DEST", help="The destination cache") options = parser.parse_args() gene = TileGeneration(options.config, options) assert gene.config_file config = gene.get_config(gene.config_file) if options.layer: copy = Copy() copy.copy(options, gene, options.layer, options.source, options.dest, "copy") else: layers = (config.config["generation"]["default_layers"] if "default_layers" in config.config["generation"] else config.config["layers"].keys()) for layer in layers: copy = Copy() copy.copy(options, gene, layer, options.source, options.dest, "copy") except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") if os.environ.get("TESTS", "false").lower() == "true": raise sys.exit(1)
def get_wmts_capabilities(gene: TileGeneration, cache_name: str, exit_: bool = False) -> Optional[str]: """Get the WMTS capabilities for a configuration file.""" assert gene.config_file config = gene.get_config(gene.config_file) cache = config.config["caches"][cache_name] if _validate_generate_wmts_capabilities(cache, cache_name, exit_): server = gene.get_main_config().config.get("server") base_urls = _get_base_urls(cache) _fill_legend(gene, cache, server, base_urls) data = pkgutil.get_data("tilecloud_chain", "wmts_get_capabilities.jinja") assert data return cast( str, jinja2_template( data.decode("utf-8"), layers=config.config["layers"], layer_legends=gene.layer_legends, grids=config.config["grids"], getcapabilities=urljoin( # type: ignore base_urls[0], (server.get("wmts_path", "wmts") + "/1.0.0/WMTSCapabilities.xml" if server is not None else cache.get("wmtscapabilities_file", "1.0.0/WMTSCapabilities.xml")), ), base_urls=base_urls, base_url_postfix=(server.get("wmts_path", "wmts") + "/") if server is not None else "", get_tile_matrix_identifier=get_tile_matrix_identifier, server=server is not None, has_metadata="metadata" in config.config, metadata=config.config.get("metadata"), has_provider="provider" in config.config, provider=config.config.get("provider"), enumerate=enumerate, ceil=math.ceil, int=int, sorted=sorted, ), ) return None
def get_status(gene: TileGeneration) -> List[str]: """Get the tile generation status.""" config = gene.get_main_config() store = get_queue_store(config, False) prefix = "redis" if "redis" in config.config else "sqs" conf = config.config[ "redis"] if "redis" in config.config else config.config["sqs"] stats_prefix = [prefix, conf["queue"]] with stats.timer_context(stats_prefix + ["get_stats"]): status_ = store.get_status() return [name + ": " + str(value) for name, value in status_.items()]
def test_near(self): class Opt: zoom = '3' test = 196 near = [600000.0, 200000.0] quiet = False verbose = False debug = False time = None logging_configuration_file = None gene = TileGeneration('tilecloud_chain/tests/tilegeneration/test.yaml', Opt(), 'point') self.assertEqual(gene.geoms[3].bounds, (583840.0, 173360.0, 624800.0, 214320.0))
def process(): parser = ArgumentParser( description='Used to copy the tiles from a cache to an other', prog='./buildout/bin/generate_copy') add_comon_options(parser, near=False, time=False, dimensions=True) parser.add_argument('process', metavar="PROCESS", help='The process name to do') options = parser.parse_args() gene = TileGeneration(options.config, options) if (options.layer): # pragma: no cover copy = Copy() copy.copy(options, gene, options.layer, options.cache, options.cache, 'process') else: for layer in gene.config['generation']['default_layers']: copy = Copy() copy.copy(options, gene, layer, options.cache, options.cache, 'process')
def process(): parser = ArgumentParser( description="Used to copy the tiles from a cache to an other", prog=sys.argv[0]) add_comon_options(parser, near=False, time=False, dimensions=True) parser.add_argument("process", metavar="PROCESS", help="The process name to do") options = parser.parse_args() gene = TileGeneration(options.config, options, multi_thread=False) copy = Copy() if options.layer: # pragma: no cover copy.copy(options, gene, options.layer, options.cache, options.cache, "process") else: layers_name = (gene.config["generation"]["default_layers"] if "default_layers" in gene.config.get( "generation", {}) else gene.layers.keys()) for layer in layers_name: copy.copy(options, gene, layer, options.cache, options.cache, "process")
def process(): parser = ArgumentParser( description='Used to copy the tiles from a cache to an other', prog=sys.argv[0]) add_comon_options(parser, near=False, time=False, dimensions=True) parser.add_argument('process', metavar="PROCESS", help='The process name to do') options = parser.parse_args() gene = TileGeneration(options.config, options) copy = Copy() if options.layer: # pragma: no cover copy.copy(options, gene, options.layer, options.cache, options.cache, 'process') else: layers_name = gene.config['generation']['default_layers'] \ if 'default_layers' in gene.config.get('generation', {}) \ else gene.layers.keys() for layer in layers_name: copy.copy(options, gene, layer, options.cache, options.cache, 'process')
def init_tilegeneration(config_file): global tilegeneration # pylint: disable=global-statement if tilegeneration is None: logger.info("Config file: '%s'", config_file) log_level = os.environ.get("TILE_SERVER_LOGLEVEL") tilegeneration = TileGeneration( config_file, collections.namedtuple( "Options", ["verbose", "debug", "quiet", "bbox", "zoom", "test", "near", "time", "geom"], )( log_level == "verbose", log_level == "debug", log_level == "quiet", None, None, None, None, None, True, ), configure_logging=False, multi_thread=False, )
def main(): parser = ArgumentParser(description='Used to generate the tiles', prog='./buildout/bin/generate_tiles') add_comon_options(parser, dimensions=True) parser.add_argument( '--get-hash', metavar="TILE", help='get the empty tiles hash, use the specified TILE z/x/y') parser.add_argument( '--get-bbox', metavar="TILE", help= 'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument( '--role', default='local', choices=('local', 'master', 'slave'), help='local/master/slave, master to file the queue and ' 'slave to generate the tiles') parser.add_argument('--daemonize', default=False, action="store_true", help='run as a daemon') parser.add_argument( '--tiles', metavar="FILE", help= 'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument('--generated-tiles-file', metavar="FILE", help='Store the tiles in a file (unrecommended)') options = parser.parse_args() if options.daemonize: daemonize() # pragma: no cover gene = TileGeneration(options.config, options) if options.get_hash is None and options.get_bbox is None and \ 'authorised_user' in gene.config['generation'] and \ gene.config['generation']['authorised_user'] != getuser(): exit('not authorised, authorised user is: %s.' % gene.config['generation']['authorised_user']) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.tiles is not None and options.role not in [ 'local', 'master' ]: # pragma: no cover exit("The --tiles option work only with role local or master") try: if (options.layer): generate = Generate() generate.gene(options, gene, options.layer) elif options.get_bbox: # pragma: no cover exit("With --get-bbox option we needs to specify a layer") elif options.get_hash: # pragma: no cover exit("With --get-hash option we needs to specify a layer") elif options.tiles: # pragma: no cover exit("With --tiles option we needs to specify a layer") else: for layer in gene.config['generation']['default_layers']: generate = Generate() generate.gene(options, gene, layer) finally: if gene.error_file is not None: gene.error_file.close()
def test_validate_type(self): class Opt: quiet = False verbose = False debug = False test = 0 zoom = None gene = TileGeneration('tilegeneration/test.yaml', Opt()) obj = {'value': 1} self.assertEquals(gene.validate(obj, 'object', 'value', int), False) self.assertEquals(obj['value'], 1) obj = {'value': 1.0} self.assertEquals(gene.validate(obj, 'object', 'value', int), True) obj = {'value': '1 + 1'} self.assertEquals(gene.validate(obj, 'object', 'value', int), False) self.assertEquals(obj['value'], 2) obj = {'value': '1 * 1.5'} self.assertEquals(gene.validate(obj, 'object', 'value', int), False) self.assertEquals(obj['value'], 2) obj = {'value': 'a'} self.assertEquals(gene.validate(obj, 'object', 'value', int), True) obj = {'value': {}} self.assertEquals(gene.validate(obj, 'object', 'value', int), True) obj = {'value': []} self.assertEquals(gene.validate(obj, 'object', 'value', int), True) obj = {'value': 1} self.assertEquals(gene.validate(obj, 'object', 'value', float), False) self.assertEquals(obj['value'], 1.0) obj = {'value': 1.0} self.assertEquals(gene.validate(obj, 'object', 'value', float), False) self.assertEquals(obj['value'], 1.0) obj = {'value': '1 + 1'} self.assertEquals(gene.validate(obj, 'object', 'value', float), False) self.assertEquals(obj['value'], 2.0) obj = {'value': '1 * 1.5'} self.assertEquals(gene.validate(obj, 'object', 'value', float), False) self.assertEquals(obj['value'], 1.5) obj = {'value': 'a'} self.assertEquals(gene.validate(obj, 'object', 'value', float), True) obj = {'value': {}} self.assertEquals(gene.validate(obj, 'object', 'value', float), True) obj = {'value': []} self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
def main(): parser = ArgumentParser( description='Used to generate the contextual file like the capabilities, the legends, ' 'the Apache and MapCache configuration', prog='./buildout/bin/generate_controller' ) add_comon_options(parser, tile_pyramid=False, no_geom=False) parser.add_argument( '--capabilities', '--generate_wmts_capabilities', default=False, action="store_true", help='Generate the WMTS Capabilities' ) parser.add_argument( '--legends', '--generate_legend_images', default=False, action="store_true", dest='legends', help='Generate the legend images' ) parser.add_argument( '--openlayers', '--generate-openlayers-test-page', default=False, action="store_true", dest='openlayers', help='Generate openlayers test page' ) parser.add_argument( '--mapcache', '--generate-mapcache-config', default=False, action="store_true", dest='mapcache', help='Generate MapCache configuration file' ) parser.add_argument( '--apache', '--generate-apache-config', default=False, action="store_true", dest='apache', help='Generate Apache configuration file' ) parser.add_argument( '--dump-config', default=False, action="store_true", help='Dump the used config with default values and exit' ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.dump_config: for layer in gene.config['layers'].keys(): gene.set_layer(layer, options) validate_calculate_cost(gene) _validate_generate_wmts_capabilities(gene, gene.caches[options.cache]) gene.validate_mapcache_config() gene.validate_apache_config() _validate_generate_openlayers(gene) for grid in gene.config['grids'].values(): if 'obj' in grid: del grid['obj'] print yaml.dump(gene.config) sys.exit(0) if options.legends: _generate_legend_images(gene) if options.capabilities: _generate_wmts_capabilities(gene) if options.mapcache: _generate_mapcache_config(gene) if options.apache: _generate_apache_config(gene) if options.openlayers: _generate_openlayers(gene)
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog='./buildout/bin/generate_amazon' ) add_comon_options(parser) parser.add_argument( '--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file' ) parser.add_argument( '--status', default=False, action="store_true", help='display the SQS queue status and exit' ) parser.add_argument( '--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation' ) parser.add_argument( '--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code' ) parser.add_argument( '--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database' ) parser.add_argument( '--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling' ) parser.add_argument( '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation' ) parser.add_argument( '--host', default=None, help='The host used to generate tiles' ) parser.add_argument( '--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.' ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print "EC2 not configured" sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if options.geodata and 'geodata_folder' in gene.config['ec2']: print "==== Sync geodata ====" ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code: print "==== Sync and build code ====" error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True) if error: exit(1) # pragma: no cover cmd = ['rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run_remote(cmd, host, project_dir, gene) if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']: run_remote( 'echo %s > %s' % ( gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config'] ), host, project_dir, gene ) run_remote('sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene ) ) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce( lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta() ) / len(times) ** 2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print '==== Time results ====' print 'A tile is generated in: %0.3f [ms]' % mean_time_ms print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb) print '''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb) if options.shutdown: # pragma: no cover run_remote('sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue: # pragma: no cover print "==== Till queue ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master']) project_dir = gene.config['ec2']['code_folder'] run_remote( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene ) if options.tiles_gen: # pragma: no cover print "==== Generate tiles ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'slave']) arguments.append("--daemonize") project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene) ) if options.shutdown: for p in processes: p.communicate() # wait process end else: print 'Tile generation started in background' if options.shutdown: run_remote('sudo shutdown 0') if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller")
def main(): stats.init_backends({}) parser = ArgumentParser(description="Used to generate the tiles", prog=sys.argv[0]) add_comon_options(parser, dimensions=True) parser.add_argument( "--get-hash", metavar="TILE", help="get the empty tiles hash, use the specified TILE z/x/y") parser.add_argument( "--get-bbox", metavar="TILE", help= "get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles", ) parser.add_argument( "--role", default="local", choices=("local", "master", "slave"), help= "local/master/slave, master to file the queue and slave to generate the tiles", ) parser.add_argument("--local-process-number", default=None, help="The number of process that we run in parallel") parser.add_argument("--detach", default=False, action="store_true", help="run detached from the terminal") parser.add_argument("--daemon", default=False, action="store_true", help="run continuously as a daemon") parser.add_argument( "--tiles", metavar="FILE", help= "Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles", ) options = parser.parse_args() if options.detach: detach() # pragma: no cover gene = TileGeneration(options.config, options, multi_thread=options.get_hash is None) if (options.get_hash is None and options.get_bbox is None and "authorised_user" in gene.config["generation"] and gene.config["generation"]["authorised_user"] != getuser()): sys.exit("not authorised, authorised user is: {}.".format( gene.config["generation"]["authorised_user"])) if options.cache is None: options.cache = gene.config["generation"]["default_cache"] if options.tiles is not None and options.role not in [ "local", "master" ]: # pragma: no cover sys.exit("The --tiles option work only with role local or master") try: generate = Generate(options, gene) if options.role == "slave": generate.gene() elif options.layer: generate.gene(gene.layers[options.layer]) elif options.get_bbox: # pragma: no cover sys.exit("With --get-bbox option you need to specify a layer") elif options.get_hash: # pragma: no cover sys.exit("With --get-hash option you need to specify a layer") elif options.tiles: # pragma: no cover sys.exit("With --tiles option you need to specify a layer") else: for layer in gene.config["generation"].get("default_layers", gene.layers.keys()): generate.gene(gene.layers[layer]) finally: gene.close()
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog=sys.argv[0] ) add_comon_options(parser) parser.add_argument( '--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file' ) parser.add_argument( '--status', default=False, action="store_true", help='display the SQS queue status and exit' ) parser.add_argument( '--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation' ) parser.add_argument( '--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code' ) parser.add_argument( '--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database' ) parser.add_argument( '--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling' ) parser.add_argument( '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation' ) parser.add_argument( '--host', default=None, help='The host used to generate tiles' ) parser.add_argument( '--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.' ) parser.add_argument( '--wait', default=False, action="store_true", help='Wait that all the tasks will finish.' ) parser.add_argument( '--local', default=False, action="store_true", help='Run the generation locally' ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print("EC2 not configured") sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']: print("==== Sync geodata ====") ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code and not options.local: print("==== Sync and build code ====") error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True) if error: exit(1) # pragma: no cover cmd = ['rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run(options, cmd % environ, host, project_dir, gene) if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']: run( options, 'echo %s > %s' % ( gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config'] ), host, project_dir, gene ) run(options, 'sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database and not options.local: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata and not options.local: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = None if options.local else gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( "%sgenerate_tiles %s" % ( _get_path(), ' '.join([str(a) for a in arguments]) ), host, project_dir, gene ) ) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) if PY3: results = [r.decode('utf-8') for r in results] results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce( lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta() ) / len(times) ** 2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print('==== Time results ====') print('A tile is generated in: %0.3f [ms]' % mean_time_ms) print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)) print('''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)) if options.shutdown: # pragma: no cover run(options, 'sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue and not options.local: # pragma: no cover print("==== Till queue ====") # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master', '--quiet']) project_dir = gene.config['ec2']['code_folder'] run_remote_process( options, "%sgenerate_tiles %s" % ( _get_path(), ' '.join([str(a) for a in arguments]) ), host, project_dir, gene ) sleep(5) attributes = gene.get_sqs_queue().get_attributes() print( "\rTiles to generate: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], ) ) if options.tiles_gen: # pragma: no cover print("==== Generate tiles ====") if options.wait and not options.local: print("") class Status(Thread): def run(self): # pragma: no cover while True: attributes = gene.get_sqs_queue().get_attributes() print( "\rTiles to generate/generating: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], ) ) sleep(1) status_thread = Status() status_thread.setDaemon(True) status_thread.start() arguments = _get_arguments(options) arguments.extend(['--quiet']) if not options.local: arguments.extend(['--role', 'slave']) project_dir = None if options.local else gene.config['ec2']['code_folder'] threads = [] for i in range(gene.config['ec2']['number_process']): if options.local: threads.append(run_local_process( "%sgenerate_tiles --local-process-number %i %s" % ( _get_path(), i, ' '.join([str(a) for a in arguments]) ) )) else: run_remote_process( "%sgenerate_tiles %s" % ( _get_path(), ' '.join([str(a) for a in arguments]) ), host, project_dir, gene ) print('Tile generation started') if options.shutdown: run(options, 'sudo shutdown 0') if options.wait and options.local: while len(threads) > 0: threads = [t for t in threads if t.is_alive()] sleep(1) if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller" )
def main(): parser = ArgumentParser( description="Used to calculate the generation cost", prog=sys.argv[0]) add_comon_options(parser, tile_pyramid=False, dimensions=True) parser.add_argument( "--cost-algo", "--calculate-cost-algorithm", default="area", dest="cost_algo", choices=("area", "count"), help= "The algorithm use to calculate the cost default base on the 'area' " "of the generation geometry, can also be 'count', to be base on number of tiles to generate.", ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer, base_config={"cost": {}}, multi_thread=False) all_size = 0 tile_size = 0 all_tiles = 0 if options.layer: layer = gene.layers[options.layer] (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, layer, options) tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024) else: all_time = timedelta() all_price = 0 for layer_name in gene.config["generation"]["default_layers"]: print("") print("===== {} =====".format(layer_name)) layer = gene.layers[layer_name] gene.init_layer(layer, options) (size, time, price, tiles) = _calculate_cost(gene, layer, options) tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024) all_time += time all_price += price all_size += size all_tiles += tiles print("") print("===== GLOBAL =====") print("Total number of tiles: {}".format(all_tiles)) print("Total generation time: {} [d h:mm:ss]".format( (duration_format(all_time)))) print("Total generation cost: {0:0.2f} [$]".format(all_price)) print("") print("S3 Storage: {0:0.2f} [$/month]".format( all_size * gene.config["cost"]["s3"]["storage"] / (1024.0 * 1024 * 1024))) print("S3 get: {0:0.2f} [$/month]".format( (gene.config["cost"]["s3"]["get"] * gene.config["cost"]["request_per_layers"] / 10000.0 + gene.config["cost"]["s3"]["download"] * gene.config["cost"]["request_per_layers"] * tile_size))) # if 'cloudfront' in gene.config['cost']: # print('CloudFront: %0.2f [$/month]' % () # gene.config['cost']['cloudfront']['get'] * # gene.config['cost']['request_per_layers'] / 10000.0 + # gene.config['cost']['cloudfront']['download'] * # gene.config['cost']['request_per_layers'] * tile_size) sys.exit(0)
def __init__(self, config_file): self.filters = {} self.max_zoom_seed = {} logger.info("Config file: '%s'" % config_file) self.tilegeneration = TileGeneration(config_file) self.expires_hours = self.tilegeneration.config['apache']['expires'] self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \ if 'static_allow_extension' in self.tilegeneration.config['server'] \ else ['jpeg', 'png', 'xml', 'js', 'html', 'css'] self.cache = self.tilegeneration.caches[ self.tilegeneration.config['server']['cache'] if 'cache' in self.tilegeneration.config['server'] else self.tilegeneration.config['generation']['default_cache'] ] if self.cache['type'] == 's3': # pragma: no cover s3bucket = S3Connection().bucket(self.cache['bucket']) def _get(self, path, **kwargs): global s3bucket try: s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path)) responce = s3key.get() return responce.body, responce.headers['Content-Type'] except: s3bucket = S3Connection().bucket(self.cache['bucket']) s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path)) responce = s3key.get() return responce.body, responce.headers['Content-Type'] else: folder = self.cache['folder'] or '' def _get(self, path, **kwargs): if path.split('.')[-1] not in self.static_allow_extension: # pragma: no cover return self.error(403, "Extension not allowed", **kwargs), None p = os.path.join(folder, path) if not os.path.isfile(p): # pragma: no cover return self.error(404, path + " not found", **kwargs), None with open(p, 'rb') as file: data = file.read() mime = mimetypes.guess_type(p) return data, mime[0] # get capabilities or other static files self._get = types.MethodType(_get, self) mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \ 'mapcache_base' in self.tilegeneration.config['server'] else \ 'http://localhost/' self.mapcache_baseurl = mapcache_base + self.tilegeneration.config['mapcache']['location'] + '/wmts' self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \ 'mapcache_headers' in self.tilegeneration.config['server'] else {} geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \ 'geoms_redirect' in self.tilegeneration.config['server'] else False self.layers = self.tilegeneration.config['server']['layers'] if \ 'layers' in self.tilegeneration.config['server'] else \ self.tilegeneration.layers.keys() self.stores = {} for layer_name in self.layers: layer = self.tilegeneration.layers[layer_name] # build geoms redirect if geoms_redirect: self.filters[layer_name] = self.tilegeneration.get_geoms_filter( layer=layer, grid=layer['grid_ref'], geoms=self.tilegeneration.get_geoms( layer, extent=layer['bbox'] if 'bbox' in layer else layer['grid_ref']['bbox'], ), ) if 'min_resolution_seed' in layer: max_zoom_seed = -1 for zoom, resolution in enumerate(layer['grid_ref']['resolutions']): if resolution > layer['min_resolution_seed']: max_zoom_seed = zoom self.max_zoom_seed[layer_name] = max_zoom_seed else: self.max_zoom_seed[layer_name] = 999999 # build stores store_defs = [{ 'ref': [layer_name], 'dimensions': {}, }] for dimension in layer['dimensions']: new_store_defs = [] for store_def in store_defs: for value in dimension['values']: dimensions = {} dimensions.update(store_def['dimensions']) dimensions[dimension['name']] = value new_store_defs.append({ 'ref': store_def['ref'] + [value], 'dimensions': dimensions, }) store_defs = new_store_defs for store_def in store_defs: self.stores['/'.join(store_def['ref'])] = \ self.tilegeneration.get_store(self.cache, layer, store_def['dimensions'], read_only=True)
def main(): stats.init_backends({}) parser = ArgumentParser(description='Used to generate the tiles', prog=sys.argv[0]) add_comon_options(parser, dimensions=True) parser.add_argument( '--get-hash', metavar="TILE", help='get the empty tiles hash, use the specified TILE z/x/y' ) parser.add_argument( '--get-bbox', metavar="TILE", help='get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument( '--role', default='local', choices=('local', 'master', 'slave'), help='local/master/slave, master to file the queue and ' 'slave to generate the tiles' ) parser.add_argument( "--local-process-number", default=None, help="The number of process that we run in parallel" ) parser.add_argument( '--detach', default=False, action="store_true", help='run detached from the terminal' ) parser.add_argument( '--daemon', default=False, action="store_true", help='run continuously as a daemon' ) parser.add_argument( '--tiles', metavar="FILE", help='Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles' ) options = parser.parse_args() if options.detach: detach() # pragma: no cover gene = TileGeneration(options.config, options) if options.get_hash is None and options.get_bbox is None and \ 'authorised_user' in gene.config['generation'] and \ gene.config['generation']['authorised_user'] != getuser(): exit('not authorised, authorised user is: {}.'.format(gene.config['generation']['authorised_user'])) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.tiles is not None and options.role not in ['local', 'master']: # pragma: no cover exit("The --tiles option work only with role local or master") try: generate = Generate(options, gene) if options.role == 'slave': generate.gene() elif options.layer: generate.gene(gene.layers[options.layer]) elif options.get_bbox: # pragma: no cover exit("With --get-bbox option you need to specify a layer") elif options.get_hash: # pragma: no cover exit("With --get-hash option you need to specify a layer") elif options.tiles: # pragma: no cover exit("With --tiles option you need to specify a layer") else: for layer in gene.config['generation'].get('default_layers', gene.layers.keys()): generate.gene(gene.layers[layer]) finally: gene.close()
def test_validate_type(self): class Opt: quiet = False verbose = False debug = False test = 0 zoom = None gene = TileGeneration("tilegeneration/test.yaml", Opt()) obj = {"value": 1} self.assertEqual(gene.validate(obj, "object", "value", int), False) self.assertEqual(obj["value"], 1) obj = {"value": 1.0} self.assertEqual(gene.validate(obj, "object", "value", int), True) obj = {"value": "1 + 1"} self.assertEqual(gene.validate(obj, "object", "value", int), False) self.assertEqual(obj["value"], 2) obj = {"value": "1 * 1.5"} self.assertEqual(gene.validate(obj, "object", "value", int), False) self.assertEqual(obj["value"], 2) obj = {"value": "a"} self.assertEqual(gene.validate(obj, "object", "value", int), True) obj = {"value": {}} self.assertEqual(gene.validate(obj, "object", "value", int), True) obj = {"value": []} self.assertEqual(gene.validate(obj, "object", "value", int), True) obj = {"value": 1} self.assertEqual(gene.validate(obj, "object", "value", float), False) self.assertEqual(obj["value"], 1.0) obj = {"value": 1.0} self.assertEqual(gene.validate(obj, "object", "value", float), False) self.assertEqual(obj["value"], 1.0) obj = {"value": "1 + 1"} self.assertEqual(gene.validate(obj, "object", "value", float), False) self.assertEqual(obj["value"], 2.0) obj = {"value": "1 * 1.5"} self.assertEqual(gene.validate(obj, "object", "value", float), False) self.assertEqual(obj["value"], 1.5) obj = {"value": "a"} self.assertEqual(gene.validate(obj, "object", "value", float), True) obj = {"value": {}} self.assertEqual(gene.validate(obj, "object", "value", float), True) obj = {"value": []} self.assertEqual(gene.validate(obj, "object", "value", float), True)
def __init__(self, config_file): self.filters = {} self.max_zoom_seed = {} logger.info("Config file: '{}'".format(config_file)) self.tilegeneration = TileGeneration(config_file) self.expires_hours = self.tilegeneration.config['apache']['expires'] self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \ if 'static_allow_extension' in self.tilegeneration.config['server'] \ else ['jpeg', 'png', 'xml', 'js', 'html', 'css'] self.cache = self.tilegeneration.caches[ self.tilegeneration.config['server']['cache'] if 'cache' in self.tilegeneration.config['server'] else self.tilegeneration. config['generation']['default_cache']] if self.cache['type'] == 's3': # pragma: no cover client = tilecloud.store.s3.get_client(self.cache.get('host')) bucket = self.cache['bucket'] def _get(self, path, **kwargs): key_name = os.path.join('{folder}'.format(**self.cache), path) try: response = client.get_object(Bucket=bucket, Key=key_name) return response['Body'].read(), response.get('ContentType') except Exception: client = tilecloud.store.s3.get_client( self.cache.get('host')) response = client.get_object(Bucket=bucket, Key=key_name) return response['Body'].read(), response.get('ContentType') else: folder = self.cache['folder'] or '' def _get(self, path, **kwargs): if path.split( '.' )[-1] not in self.static_allow_extension: # pragma: no cover return self.error(403, "Extension not allowed", **kwargs), None p = os.path.join(folder, path) if not os.path.isfile(p): # pragma: no cover return self.error(404, path + " not found", **kwargs), None with open(p, 'rb') as file: data = file.read() mime = mimetypes.guess_type(p) return data, mime[0] # get capabilities or other static files self._get = types.MethodType(_get, self) mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \ 'mapcache_base' in self.tilegeneration.config['server'] else \ 'http://localhost/' self.mapcache_baseurl = mapcache_base + self.tilegeneration.config[ 'mapcache']['location'] + '/wmts' self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \ 'mapcache_headers' in self.tilegeneration.config['server'] else {} geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \ 'geoms_redirect' in self.tilegeneration.config['server'] else False self.layers = self.tilegeneration.config['server']['layers'] if \ 'layers' in self.tilegeneration.config['server'] else \ self.tilegeneration.layers.keys() self.stores = {} for layer_name in self.layers: layer = self.tilegeneration.layers[layer_name] # build geoms redirect if geoms_redirect: self.filters[ layer_name] = self.tilegeneration.get_geoms_filter( layer=layer, grid=layer['grid_ref'], geoms=self.tilegeneration.get_geoms( layer, extent=layer['bbox'] if 'bbox' in layer else layer['grid_ref']['bbox'], ), ) if 'min_resolution_seed' in layer: max_zoom_seed = -1 for zoom, resolution in enumerate( layer['grid_ref']['resolutions']): if resolution > layer['min_resolution_seed']: max_zoom_seed = zoom self.max_zoom_seed[layer_name] = max_zoom_seed else: self.max_zoom_seed[layer_name] = 999999 # build stores store_defs = [{ 'ref': [layer_name], 'dimensions': {}, }] for dimension in layer['dimensions']: new_store_defs = [] for store_def in store_defs: for value in dimension['values']: dimensions = {} dimensions.update(store_def['dimensions']) dimensions[dimension['name']] = value new_store_defs.append({ 'ref': store_def['ref'] + [value], 'dimensions': dimensions, }) store_defs = new_store_defs for store_def in store_defs: self.stores['/'.join(store_def['ref'])] = \ self.tilegeneration.get_store(self.cache, layer, read_only=True)
def _calculate_cost(gene: TileGeneration, layer_name: str, options: Namespace) -> Tuple[float, timedelta, float, int]: nb_metatiles = {} nb_tiles = {} config = gene.get_config(options.config) layer = config.config["layers"][layer_name] meta = layer["meta"] if options.cost_algo == "area": tile_size = config.config["grids"][layer["grid"]]["tile_size"] for zoom, resolution in enumerate( config.config["grids"][layer["grid"]]["resolutions"]): if "min_resolution_seed" in layer and resolution < layer[ "min_resolution_seed"]: continue print(f"Calculate zoom {zoom}.") px_buffer = layer["px_buffer"] + layer["meta_buffer"] if meta else 0 m_buffer = px_buffer * resolution if meta: size = tile_size * layer["meta_size"] * resolution meta_buffer = size * 0.7 + m_buffer meta_geom = gene.get_geoms(config, layer_name)[zoom].buffer( meta_buffer, 1) nb_metatiles[zoom] = int(round(meta_geom.area / size**2)) size = tile_size * resolution tile_buffer = size * 0.7 + m_buffer geom = gene.get_geoms(config, layer_name)[zoom].buffer(tile_buffer, 1) nb_tiles[zoom] = int(round(geom.area / size**2)) elif options.cost_algo == "count": gene.init_tilecoords(config, layer_name) gene.add_geom_filter() if meta: def count_metatile(tile: Tile) -> Tile: if tile: if tile.tilecoord.z in nb_metatiles: nb_metatiles[tile.tilecoord.z] += 1 else: nb_metatiles[tile.tilecoord.z] = 1 return tile gene.imap(count_metatile) class MetaTileSplitter(TileStore): """Convert the metatile flow to tile flow.""" @staticmethod def get(tiles: Iterable[Tile]) -> Iterator[Tile]: for metatile in tiles: for tilecoord in metatile.tilecoord: yield Tile(tilecoord) gene.add_metatile_splitter(MetaTileSplitter()) # Only keep tiles that intersect geometry gene.add_geom_filter() def count_tile(tile: Tile) -> Tile: if tile: if tile.tilecoord.z in nb_tiles: nb_tiles[tile.tilecoord.z] += 1 else: print(f"Calculate zoom {tile.tilecoord.z}.") nb_tiles[tile.tilecoord.z] = 1 return tile gene.imap(count_tile) run = Run(gene, gene.functions_metatiles) assert gene.tilestream for tile in gene.tilestream: tile.metadata["layer"] = layer_name run(tile) times = {} print() for z, nb_metatile in nb_metatiles.items(): print(f"{nb_metatile} meta tiles in zoom {z}.") times[z] = layer["cost"]["metatile_generation_time"] * nb_metatile price: float = 0 all_size: float = 0 all_time: float = 0 all_tiles = 0 for z, nb_tile in nb_tiles.items(): print() print(f"{nb_tile} tiles in zoom {z}.") all_tiles += nb_tile if meta: time = times[z] + layer["cost"]["tile_generation_time"] * nb_tile else: time = layer["cost"]["tileonly_generation_time"] * nb_tile size = layer["cost"]["tile_size"] * nb_tile all_size += size all_time += time td = timedelta(milliseconds=time) print(f"Time to generate: {duration_format(td)} [d h:mm:ss]") c = gene.get_main_config( ).config["cost"]["s3"]["put"] * nb_tile / 1000.0 price += c print(f"S3 PUT: {c:0.2f} [$]") if "sqs" in gene.get_main_config().config: if meta: nb_sqs = nb_metatiles[z] * 3 else: nb_sqs = nb_tile * 3 c = nb_sqs * gene.get_main_config( ).config["cost"]["sqs"]["request"] / 1000000.0 price += c print(f"SQS usage: {c:0.2f} [$]") print() td = timedelta(milliseconds=all_time) print(f"Number of tiles: {all_tiles}") print(f"Generation time: {duration_format(td)} [d h:mm:ss]") print(f"Generation cost: {price:0.2f} [$]") return all_size, td, price, all_tiles
def __init__(self, config_file): self.filters = {} self.max_zoom_seed = {} logger.info("Config file: '%s'" % config_file) self.tilegeneration = TileGeneration(config_file) if not self.tilegeneration.validate_apache_config(): # pragma: no cover raise "Apache configuration error" self.expires_hours = self.tilegeneration.config['apache']['expires'] self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \ if 'static_allow_extension' in self.tilegeneration.config['server'] \ else ['jpeg', 'png', 'xml', 'js', 'html', 'css'] self.cache = self.tilegeneration.caches[ self.tilegeneration.config['server']['cache'] if 'cache' in self.tilegeneration.config['server'] else self.tilegeneration.config['generation']['default_cache'] ] if self.cache['type'] == 's3': # pragma: no cover s3bucket = S3Connection().bucket(self.cache['bucket']) def _get(self, path, **kwargs): global s3bucket try: s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path)) responce = s3key.get() return responce.body, responce.headers['Content-Type'] except: s3bucket = S3Connection().bucket(self.cache['bucket']) s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path)) responce = s3key.get() return responce.body, responce.headers['Content-Type'] else: folder = self.cache['folder'] or '' def _get(self, path, **kwargs): if path.split('.')[-1] not in self.static_allow_extension: # pragma: no cover return self.error(403, "Extension not allowed", **kwargs), None p = os.path.join(folder, path) if not os.path.isfile(p): # pragma: no cover return self.error(404, path + " not found", **kwargs), None with open(p, 'rb') as file: data = file.read() mime = mimetypes.guess_type(p) return data, mime[0] # get capabilities or other static files self._get = types.MethodType(_get, self) if not self.tilegeneration.validate_mapcache_config(): # pragma: no cover raise "Mapcache configuration error" mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \ 'mapcache_base' in self.tilegeneration.config['server'] else \ 'http://localhost/' self.mapcache_baseurl = mapcache_base + self.tilegeneration.config['mapcache']['location'] + '/wmts' self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \ 'mapcache_headers' in self.tilegeneration.config['server'] else {} geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \ 'geoms_redirect' in self.tilegeneration.config['server'] else False self.layers = self.tilegeneration.config['server']['layers'] if \ 'layers' in self.tilegeneration.config['server'] else \ self.tilegeneration.layers.keys() self.stores = {} for layer_name in self.layers: layer = self.tilegeneration.layers[layer_name] # build geoms redirect if geoms_redirect: self.filters[layer_name] = self.tilegeneration.get_geoms_filter( layer=layer, grid=layer['grid_ref'], geoms=self.tilegeneration.get_geoms( layer, extent=layer['bbox'] if 'bbox' in layer else layer['grid_ref']['bbox'], ), ) if 'min_resolution_seed' in layer: max_zoom_seed = -1 for zoom, resolution in enumerate(layer['grid_ref']['resolutions']): if resolution > layer['min_resolution_seed']: max_zoom_seed = zoom self.max_zoom_seed[layer_name] = max_zoom_seed else: self.max_zoom_seed[layer_name] = sys.maxint # build stores store_defs = [{ 'ref': [layer_name], 'dimensions': [], }] for dimension in layer['dimensions']: new_store_defs = [] for store_def in store_defs: for value in dimension['values']: new_store_defs.append({ 'ref': store_def['ref'] + [value], 'dimensions': store_def['dimensions'] + [(dimension['name'], value)], }) store_defs = new_store_defs for store_def in store_defs: self.stores['/'.join(store_def['ref'])] = \ self.tilegeneration.get_store(self.cache, layer, store_def['dimensions'], read_only=True)
def main(): parser = ArgumentParser(description='Used to generate the tiles', prog=sys.argv[0]) add_comon_options(parser, dimensions=True) parser.add_argument( '--get-hash', metavar="TILE", help='get the empty tiles hash, use the specified TILE z/x/y') parser.add_argument( '--get-bbox', metavar="TILE", help= 'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument( '--role', default='local', choices=('local', 'master', 'slave'), help='local/master/slave, master to file the queue and ' 'slave to generate the tiles') parser.add_argument("--local-process-number", default=None, help="The number of process that we run in parallel") parser.add_argument('--detach', default=False, action="store_true", help='run detached from the terminal') parser.add_argument('--daemon', default=False, action="store_true", help='run continuously as a daemon') parser.add_argument( '--tiles', metavar="FILE", help= 'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles' ) parser.add_argument('--generated-tiles-file', metavar="FILE", help='Store the tiles in a file (unrecommended)') options = parser.parse_args() if options.detach: detach() # pragma: no cover gene = TileGeneration(options.config, options) if options.get_hash is None and options.get_bbox is None and \ 'authorised_user' in gene.config['generation'] and \ gene.config['generation']['authorised_user'] != getuser(): exit('not authorised, authorised user is: {}.'.format( gene.config['generation']['authorised_user'])) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.tiles is not None and options.role not in [ 'local', 'master' ]: # pragma: no cover exit("The --tiles option work only with role local or master") try: if options.role == 'slave': generate = Generate() generate.gene(options, gene) elif (options.layer): generate = Generate() generate.gene(options, gene, options.layer) elif options.get_bbox: # pragma: no cover exit("With --get-bbox option we needs to specify a layer") elif options.get_hash: # pragma: no cover exit("With --get-hash option we needs to specify a layer") elif options.tiles: # pragma: no cover exit("With --tiles option we needs to specify a layer") else: for layer in gene.config['generation'].get('default_layers', gene.layers.keys()): generate = Generate() generate.gene(options, gene, layer) finally: if gene.error_file is not None: gene.error_file.close()
class Server: def __init__(self, config_file): self.filters = {} self.max_zoom_seed = {} logger.info("Config file: '%s'" % config_file) self.tilegeneration = TileGeneration(config_file) self.expires_hours = self.tilegeneration.config['apache']['expires'] self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \ if 'static_allow_extension' in self.tilegeneration.config['server'] \ else ['jpeg', 'png', 'xml', 'js', 'html', 'css'] self.cache = self.tilegeneration.caches[ self.tilegeneration.config['server']['cache'] if 'cache' in self.tilegeneration.config['server'] else self.tilegeneration.config['generation']['default_cache'] ] if self.cache['type'] == 's3': # pragma: no cover s3bucket = S3Connection().bucket(self.cache['bucket']) def _get(self, path, **kwargs): global s3bucket try: s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path)) responce = s3key.get() return responce.body, responce.headers['Content-Type'] except: s3bucket = S3Connection().bucket(self.cache['bucket']) s3key = s3bucket.key(os.path.join('%(folder)s' % self.cache, path)) responce = s3key.get() return responce.body, responce.headers['Content-Type'] else: folder = self.cache['folder'] or '' def _get(self, path, **kwargs): if path.split('.')[-1] not in self.static_allow_extension: # pragma: no cover return self.error(403, "Extension not allowed", **kwargs), None p = os.path.join(folder, path) if not os.path.isfile(p): # pragma: no cover return self.error(404, path + " not found", **kwargs), None with open(p, 'rb') as file: data = file.read() mime = mimetypes.guess_type(p) return data, mime[0] # get capabilities or other static files self._get = types.MethodType(_get, self) mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \ 'mapcache_base' in self.tilegeneration.config['server'] else \ 'http://localhost/' self.mapcache_baseurl = mapcache_base + self.tilegeneration.config['mapcache']['location'] + '/wmts' self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \ 'mapcache_headers' in self.tilegeneration.config['server'] else {} geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \ 'geoms_redirect' in self.tilegeneration.config['server'] else False self.layers = self.tilegeneration.config['server']['layers'] if \ 'layers' in self.tilegeneration.config['server'] else \ self.tilegeneration.layers.keys() self.stores = {} for layer_name in self.layers: layer = self.tilegeneration.layers[layer_name] # build geoms redirect if geoms_redirect: self.filters[layer_name] = self.tilegeneration.get_geoms_filter( layer=layer, grid=layer['grid_ref'], geoms=self.tilegeneration.get_geoms( layer, extent=layer['bbox'] if 'bbox' in layer else layer['grid_ref']['bbox'], ), ) if 'min_resolution_seed' in layer: max_zoom_seed = -1 for zoom, resolution in enumerate(layer['grid_ref']['resolutions']): if resolution > layer['min_resolution_seed']: max_zoom_seed = zoom self.max_zoom_seed[layer_name] = max_zoom_seed else: self.max_zoom_seed[layer_name] = 999999 # build stores store_defs = [{ 'ref': [layer_name], 'dimensions': {}, }] for dimension in layer['dimensions']: new_store_defs = [] for store_def in store_defs: for value in dimension['values']: dimensions = {} dimensions.update(store_def['dimensions']) dimensions[dimension['name']] = value new_store_defs.append({ 'ref': store_def['ref'] + [value], 'dimensions': dimensions, }) store_defs = new_store_defs for store_def in store_defs: self.stores['/'.join(store_def['ref'])] = \ self.tilegeneration.get_store(self.cache, layer, store_def['dimensions'], read_only=True) def __call__(self, environ, start_response): params = {} for key, value in parse_qs(environ['QUERY_STRING'], True).items(): params[key.upper()] = value[0] path = None if len(params) > 0 else environ['PATH_INFO'][1:].split('/') return self.serve(path, params, start_response=start_response) def serve(self, path, params, **kwargs): dimensions = [] if path is not None: if len(path) >= 1 and path[0] == 'static': body, mime = self._get('/'.join(path[1:]), **kwargs) if mime is not None: return self.responce(body, { 'Content-Type': mime, 'Expires': ( datetime.datetime.utcnow() + datetime.timedelta(hours=self.expires_hours) ).isoformat(), 'Cache-Control': "max-age=%i" % (3600 * self.expires_hours), 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET', }, **kwargs) else: # pragma: no cover return body elif len(path) >= 1 and path[0] != 'wmts': # pragma: no cover return self.error( 404, "Type '%s' don't exists, allows values: 'wmts' or 'static'" % path[0], **kwargs ) path = path[1:] # remove type if len(path) == 2 and path[0] == '1.0.0' and path[1].lower() == 'wmtscapabilities.xml': params['SERVICE'] = 'WMTS' params['VERSION'] = '1.0.0' params['REQUEST'] = 'GetCapabilities' elif len(path) < 7: return self.error(400, "Not enough path", **kwargs) else: params['SERVICE'] = 'WMTS' params['VERSION'] = path[0] params['LAYER'] = path[1] params['STYLE'] = path[2] if params['LAYER'] in self.layers: layer = self.tilegeneration.layers[params['LAYER']] else: return self.error(400, "Wrong Layer '%s'" % params['LAYER'], **kwargs) index = 3 dimensions = path[index:index + len(layer['dimensions'])] for dimension in layer['dimensions']: params[dimension['name'].upper()] = path[index] index += 1 last = path[-1].split('.') if len(path) < index + 4: # pragma: no cover return self.error(400, "Not enough path", **kwargs) params['TILEMATRIXSET'] = path[index] params['TILEMATRIX'] = path[index + 1] params['TILEROW'] = path[index + 2] if len(path) == index + 4: params['REQUEST'] = 'GetTile' params['TILECOL'] = last[0] if last[1] != layer['extension']: # pragma: no cover return self.error(400, "Wrong extention '%s'" % last[1], **kwargs) elif len(path) == index + 6: params['REQUEST'] = 'GetFeatureInfo' params['TILECOL'] = path[index + 3] params['I'] = path[index + 4] params['J'] = last[0] params['INFO_FORMAT'] = layer.get('info_formats', ['application/vnd.ogc.gml'])[0] else: # pragma: no cover return self.error(400, "Wrong path length", **kwargs) params['FORMAT'] = layer['mime_type'] else: if \ 'SERVICE' not in params or \ 'REQUEST' not in params or \ 'VERSION' not in params: return self.error(400, "Not all required parameters are present", **kwargs) if params['SERVICE'] != 'WMTS': return self.error(400, "Wrong Service '%s'" % params['SERVICE'], **kwargs) if params['VERSION'] != '1.0.0': return self.error(400, "Wrong Version '%s'" % params['VERSION'], **kwargs) if params['REQUEST'] == 'GetCapabilities': wmtscapabilities_file = self.cache['wmtscapabilities_file'] body, mime = self._get(wmtscapabilities_file, **kwargs) if mime is not None: return self.responce(body, headers={ 'Content-Type': "application/xml", 'Expires': ( datetime.datetime.utcnow() + datetime.timedelta(hours=self.expires_hours) ).isoformat(), 'Cache-Control': "max-age=%i" % (3600 * self.expires_hours), 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET', }, **kwargs) else: # pragma: no cover return body if \ 'FORMAT' not in params or \ 'LAYER' not in params or \ 'TILEMATRIXSET' not in params or \ 'TILEMATRIX' not in params or \ 'TILEROW' not in params or \ 'TILECOL' not in params: # pragma: no cover return self.error(400, "Not all required parameters are present", **kwargs) if path is None: if params['LAYER'] in self.layers: layer = self.tilegeneration.layers[params['LAYER']] else: return self.error(400, "Wrong Layer '%s'" % params['LAYER'], **kwargs) for dimension in layer['dimensions']: dimensions.append( params[dimension['name'].upper()] if dimension['name'].lower() in params else dimension['default'] ) if params['STYLE'] != layer['wmts_style']: return self.error(400, "Wrong Style '%s'" % params['STYLE'], **kwargs) if params['TILEMATRIXSET'] != layer['grid']: return self.error(400, "Wrong TileMatrixSet '%s'" % params['TILEMATRIXSET'], **kwargs) tile = Tile(TileCoord( # TODO fix for matrix_identifier = resolution int(params['TILEMATRIX']), int(params['TILECOL']), int(params['TILEROW']), )) if params['REQUEST'] == 'GetFeatureInfo': if \ 'I' not in params or \ 'J' not in params or \ 'INFO_FORMAT' not in params: # pragma: no cover return self.error(400, "Not all required parameters are present", **kwargs) if 'query_layers' in layer: return self.forward( layer['url'] + '?' + urlencode({ 'SERVICE': 'WMS', 'VERSION': '1.1.1', 'REQUEST': 'GetFeatureInfo', 'LAYERS': layer['layers'], 'QUERY_LAYERS': layer['query_layers'], 'STYLES': params['STYLE'], 'FORMAT': params['FORMAT'], 'INFO_FORMAT': params['INFO_FORMAT'], 'WIDTH': layer['grid_ref']['tile_size'], 'HEIGHT': layer['grid_ref']['tile_size'], 'SRS': layer['grid_ref']['srs'], 'BBOX': layer['grid_ref']['obj'].extent(tile.tilecoord), 'X': params['I'], 'Y': params['J'], }), no_cache=True, **kwargs ) else: # pragma: no cover return self.error(400, "Layer '%s' not queryable" % layer['name'], **kwargs) if params['REQUEST'] != 'GetTile': return self.error(400, "Wrong Request '%s'" % params['REQUEST'], **kwargs) if params['FORMAT'] != layer['mime_type']: return self.error(400, "Wrong Format '%s'" % params['FORMAT'], **kwargs) if tile.tilecoord.z > self.max_zoom_seed[layer['name']]: # pragma: no cover return self.forward( self.mapcache_baseurl + '?' + urlencode(params), headers=self.mapcache_header, **kwargs ) if layer['name'] in self.filters: layer_filter = self.filters[layer['name']] meta_size = layer['meta_size'] meta_tilecoord = TileCoord( # TODO fix for matrix_identifier = resolution tile.tilecoord.z, tile.tilecoord.x / meta_size * meta_size, tile.tilecoord.y / meta_size * meta_size, meta_size, ) if meta_size != 1 else tile.tilecoord if not layer_filter.filter_tilecoord(meta_tilecoord): # pragma: no cover return self.forward( self.mapcache_baseurl + '?' + urlencode(params), headers=self.mapcache_header, **kwargs ) store_ref = '/'.join([params['LAYER']] + dimensions) if store_ref in self.stores: # pragma: no cover store = self.stores[store_ref] else: # pragma: no cover return self.error( 400, "No store found for layer '%s' and dimensions %s" % ( layer['name'], ', '.join(dimensions) ), **kwargs ) tile = store.get_one(tile) if tile: return self.responce(tile.data, headers={ 'Content-Type': tile.content_type, 'Expires': ( datetime.datetime.utcnow() + datetime.timedelta(hours=self.expires_hours) ).isoformat(), 'Cache-Control': "max-age=%i" % (3600 * self.expires_hours), 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET', }, **kwargs) else: return self.error(204, **kwargs) def forward(self, url, headers={}, no_cache=False, **kwargs): if no_cache: headers['Cache-Control'] = 'no-cache' headers['Pragma'] = 'no-cache' responce = requests.get(url, headers=headers) if responce.status_code == 200: responce_headers = responce.headers.copy() if no_cache: responce_headers['Cache-Control'] = 'no-cache, no-store' responce_headers['Pragma'] = 'no-cache' else: # pragma: no cover responce_headers['Expires'] = ( datetime.datetime.utcnow() + datetime.timedelta(hours=self.expires_hours) ).isoformat() responce_headers['Cache-Control'] = "max-age=%i" % (3600 * self.expires_hours) responce_headers['Access-Control-Allow-Origin'] = '*' responce_headers['Access-Control-Allow-Methods'] = 'GET' responce_headers return self.responce(responce.content, headers=responce_headers, **kwargs) else: # pragma: no cover message = "The URL '%s' return '%i %s', content:\n%s" % ( url, responce.status_code, responce.reason, responce.text, ) logger.warning(message) return self.error(502, message=message, **kwargs) HTTP_MESSAGES = { 204: '204 No Content', 400: '400 Bad Request', 403: '403 Forbidden', 404: '404 Not Found', 502: '502 Bad Gateway', } def error(self, code, message='', start_response=None): start_response(self.HTTP_MESSAGES[code], []) return [message] def responce(self, data, headers={}, start_response=None): headers['Content-Length'] = str(len(data)) start_response('200 OK', headers.items()) return [data]
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog=sys.argv[0]) add_comon_options(parser) parser.add_argument('--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file') parser.add_argument('--status', default=False, action="store_true", help='display the SQS queue status and exit') parser.add_argument('--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation') parser.add_argument('--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code') parser.add_argument('--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database') parser.add_argument('--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling') parser.add_argument('--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation') parser.add_argument('--host', default=None, help='The host used to generate tiles') parser.add_argument('--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.') parser.add_argument('--wait', default=False, action="store_true", help='Wait that all the tasks will finish.') parser.add_argument('--local', default=False, action="store_true", help='Run the generation locally') options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print("EC2 not configured") sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if not options.local and options.geodata and 'geodata_folder' in gene.config[ 'ec2']: # pragma: no cover print("==== Sync geodata ====") ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code and not options.local: print("==== Sync and build code ====") cmd = [ 'rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run(options, cmd % environ, host, project_dir, gene) if 'apache_content' in gene.config[ 'ec2'] and 'apache_config' in gene.config['ec2']: run( options, 'echo %s > %s' % (gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config']), host, project_dir, gene) run(options, 'sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database and not options.local: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata and not options.local: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = None if options.local else gene.config['ec2'][ 'code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( "%sgenerate_tiles %s" % (_get_path(), ' '.join([str(a) for a in arguments])), host, project_dir, gene)) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) if PY3: results = [r.decode('utf-8') for r in results] results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce(lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta()) / len(times)**2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print('==== Time results ====') print('A tile is generated in: %0.3f [ms]' % mean_time_ms) print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)) print('''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)) if options.shutdown: # pragma: no cover run(options, 'sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue and not options.local: # pragma: no cover print("==== Till queue ====") # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master', '--quiet']) project_dir = gene.config['ec2']['code_folder'] run_remote_process( options, "%sgenerate_tiles %s" % (_get_path(), ' '.join([str(a) for a in arguments])), host, project_dir, gene) sleep(5) attributes = gene.get_sqs_queue().get_attributes() print("\rTiles to generate: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], )) if options.tiles_gen: # pragma: no cover print("==== Generate tiles ====") if options.wait and not options.local: print("") class Status(Thread): def run(self): # pragma: no cover while True: attributes = gene.get_sqs_queue().get_attributes() print("\rTiles to generate/generating: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], )) sleep(1) status_thread = Status() status_thread.setDaemon(True) status_thread.start() arguments = _get_arguments(options) arguments.extend(['--quiet']) if not options.local: arguments.extend(['--role', 'slave']) project_dir = None if options.local else gene.config['ec2'][ 'code_folder'] threads = [] for i in range(gene.config['ec2']['number_process']): if options.local: threads.append( run_local_process( "%sgenerate_tiles --local-process-number %i %s" % (_get_path(), i, ' '.join([str(a) for a in arguments])))) else: run_remote_process( "%sgenerate_tiles %s" % (_get_path(), ' '.join([str(a) for a in arguments])), host, project_dir, gene) print('Tile generation started') if options.shutdown: run(options, 'sudo shutdown 0') if options.wait and options.local: while len(threads) > 0: threads = [t for t in threads if t.is_alive()] sleep(1) if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller")
def main() -> None: """Calculate the cost, main function.""" try: parser = ArgumentParser( description="Used to calculate the generation cost", prog=sys.argv[0]) add_common_options(parser, tile_pyramid=False, dimensions=True) parser.add_argument( "--cost-algo", "--calculate-cost-algorithm", default="area", dest="cost_algo", choices=("area", "count"), help= "The algorithm use to calculate the cost default base on the 'area' " "of the generation geometry, can also be 'count', to be base on number of tiles to generate.", ) options = parser.parse_args() gene = TileGeneration( options.config, options=options, layer_name=options.layer, base_config={"cost": {}}, multi_thread=False, ) config = gene.get_config(options.config) all_size: float = 0 tile_size: float = 0 all_tiles = 0 if options.layer: layer = config.config["layers"][options.layer] (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options.layer, options) tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024) else: all_time = timedelta() all_price = 0 for layer_name in gene.get_config( options.config).config["generation"]["default_layers"]: print() print(f"===== {layer_name} =====") layer = config.config["layers"][layer_name] gene.create_log_tiles_error(layer_name) (size, time, price, tiles) = _calculate_cost(gene, layer_name, options) tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024) all_time += time all_price += price all_size += size all_tiles += tiles print() print("===== GLOBAL =====") print(f"Total number of tiles: {all_tiles}") print( f"Total generation time: {duration_format(all_time)} [d h:mm:ss]" ) print(f"Total generation cost: {all_price:0.2f} [$]") print() s3_cost = all_size * gene.get_main_config( ).config["cost"]["s3"]["storage"] / (1024.0 * 1024 * 1024) print(f"S3 Storage: {s3_cost:0.2f} [$/month]") s3_get_cost = ( gene.get_main_config().config["cost"]["s3"]["get"] * config.config["cost"]["request_per_layers"] / 10000.0 + gene.get_main_config().config["cost"]["s3"]["download"] * config.config["cost"]["request_per_layers"] * tile_size) print(f"S3 get: {s3_get_cost:0.2f} [$/month]") # if 'cloudfront' in gene.config['cost']: # print('CloudFront: %0.2f [$/month]' % () # gene.config['cost']['cloudfront']['get'] * # gene.config['cost']['request_per_layers'] / 10000.0 + # gene.config['cost']['cloudfront']['download'] * # gene.config['cost']['request_per_layers'] * tile_size) except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") sys.exit(1)
def _generate_legend_images(gene: TileGeneration) -> None: assert gene.config_file config = gene.get_config(gene.config_file) cache = config.config["caches"][gene.options.cache] for layer_name, layer in config.config["layers"].items(): if "legend_mime" in layer and "legend_extension" in layer: if layer["type"] == "wms": session = requests.session() session.headers.update(layer["headers"]) previous_hash = None for zoom, resolution in enumerate( config.config["grids"][layer["grid"]]["resolutions"]): legends = [] for wmslayer in layer["layers"].split(","): response = session.get(layer["url"] + "?" + urlencode({ "SERVICE": "WMS", "VERSION": layer.get("version", "1.0.0"), "REQUEST": "GetLegendGraphic", "LAYER": wmslayer, "FORMAT": layer["legend_mime"], "TRANSPARENT": "TRUE" if layer["legend_mime"] == "image/png" else "FALSE", "STYLE": layer["wmts_style"], "SCALE": resolution / 0.00028, })) try: legends.append( Image.open(BytesIO(response.content))) except Exception: # pragma: nocover logger.warning( "Unable to read legend image for layer '%s'-'%s', resolution '%s': %s", layer_name, wmslayer, resolution, response.content, exc_info=True, ) width = max(i.size[0] for i in legends) height = sum(i.size[1] for i in legends) image = Image.new("RGBA", (width, height)) y = 0 for i in legends: image.paste(i, (0, y)) y += i.size[1] string_io = BytesIO() image.save(string_io, FORMAT_BY_CONTENT_TYPE[layer["legend_mime"]]) result = string_io.getvalue() new_hash = sha1(result).hexdigest() # nosec if new_hash != previous_hash: previous_hash = new_hash _send( result, f"1.0.0/{layer_name}/{layer['wmts_style']}/" f"legend{zoom}.{layer['legend_extension']}", layer["legend_mime"], cache, )
class Server: def __init__(self, config_file): self.filters = {} self.max_zoom_seed = {} logger.info("Config file: '{}'".format(config_file)) self.tilegeneration = TileGeneration(config_file) self.expires_hours = self.tilegeneration.config['apache']['expires'] self.static_allow_extension = self.tilegeneration.config['server']['static_allow_extension'] \ if 'static_allow_extension' in self.tilegeneration.config['server'] \ else ['jpeg', 'png', 'xml', 'js', 'html', 'css'] self.cache = self.tilegeneration.caches[ self.tilegeneration.config['server']['cache'] if 'cache' in self.tilegeneration.config['server'] else self.tilegeneration. config['generation']['default_cache']] if self.cache['type'] == 's3': # pragma: no cover client = tilecloud.store.s3.get_client(self.cache.get('host')) bucket = self.cache['bucket'] def _get(self, path, **kwargs): key_name = os.path.join('{folder}'.format(**self.cache), path) try: response = client.get_object(Bucket=bucket, Key=key_name) return response['Body'].read(), response.get('ContentType') except Exception: client = tilecloud.store.s3.get_client( self.cache.get('host')) response = client.get_object(Bucket=bucket, Key=key_name) return response['Body'].read(), response.get('ContentType') else: folder = self.cache['folder'] or '' def _get(self, path, **kwargs): if path.split( '.' )[-1] not in self.static_allow_extension: # pragma: no cover return self.error(403, "Extension not allowed", **kwargs), None p = os.path.join(folder, path) if not os.path.isfile(p): # pragma: no cover return self.error(404, path + " not found", **kwargs), None with open(p, 'rb') as file: data = file.read() mime = mimetypes.guess_type(p) return data, mime[0] # get capabilities or other static files self._get = types.MethodType(_get, self) mapcache_base = self.tilegeneration.config['server']['mapcache_base'] if \ 'mapcache_base' in self.tilegeneration.config['server'] else \ 'http://localhost/' self.mapcache_baseurl = mapcache_base + self.tilegeneration.config[ 'mapcache']['location'] + '/wmts' self.mapcache_header = self.tilegeneration.config['server']['mapcache_headers'] if \ 'mapcache_headers' in self.tilegeneration.config['server'] else {} geoms_redirect = bool(self.tilegeneration.config['server']['geoms_redirect']) if \ 'geoms_redirect' in self.tilegeneration.config['server'] else False self.layers = self.tilegeneration.config['server']['layers'] if \ 'layers' in self.tilegeneration.config['server'] else \ self.tilegeneration.layers.keys() self.stores = {} for layer_name in self.layers: layer = self.tilegeneration.layers[layer_name] # build geoms redirect if geoms_redirect: self.filters[ layer_name] = self.tilegeneration.get_geoms_filter( layer=layer, grid=layer['grid_ref'], geoms=self.tilegeneration.get_geoms( layer, extent=layer['bbox'] if 'bbox' in layer else layer['grid_ref']['bbox'], ), ) if 'min_resolution_seed' in layer: max_zoom_seed = -1 for zoom, resolution in enumerate( layer['grid_ref']['resolutions']): if resolution > layer['min_resolution_seed']: max_zoom_seed = zoom self.max_zoom_seed[layer_name] = max_zoom_seed else: self.max_zoom_seed[layer_name] = 999999 # build stores store_defs = [{ 'ref': [layer_name], 'dimensions': {}, }] for dimension in layer['dimensions']: new_store_defs = [] for store_def in store_defs: for value in dimension['values']: dimensions = {} dimensions.update(store_def['dimensions']) dimensions[dimension['name']] = value new_store_defs.append({ 'ref': store_def['ref'] + [value], 'dimensions': dimensions, }) store_defs = new_store_defs for store_def in store_defs: self.stores['/'.join(store_def['ref'])] = \ self.tilegeneration.get_store(self.cache, layer, read_only=True) def __call__(self, environ, start_response): params = {} for key, value in parse_qs(environ['QUERY_STRING'], True).items(): params[key.upper()] = value[0] path = None if len(params) > 0 else environ['PATH_INFO'][1:].split('/') return self.serve(path, params, start_response=start_response) def serve(self, path, params, **kwargs): dimensions = [] metadata = {} if path: if len(path) >= 1 and path[0] == 'static': body, mime = self._get('/'.join(path[1:]), **kwargs) if mime is not None: return self.response( body, { 'Content-Type': mime, 'Expires': (datetime.datetime.utcnow() + datetime.timedelta( hours=self.expires_hours)).isoformat(), 'Cache-Control': "max-age={}".format((3600 * self.expires_hours)), 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET', }, **kwargs) else: # pragma: no cover return body elif len(path) >= 1 and path[0] != 'wmts': # pragma: no cover return self.error( 404, "Type '{}' don't exists, allows values: 'wmts' or 'static'" .format(path[0]), **kwargs) path = path[1:] # remove type if len(path) == 2 and path[0] == '1.0.0' and path[1].lower( ) == 'wmtscapabilities.xml': params['SERVICE'] = 'WMTS' params['VERSION'] = '1.0.0' params['REQUEST'] = 'GetCapabilities' elif len(path) < 7: return self.error(400, "Not enough path", **kwargs) else: params['SERVICE'] = 'WMTS' params['VERSION'] = path[0] params['LAYER'] = path[1] params['STYLE'] = path[2] if params['LAYER'] in self.layers: layer = self.tilegeneration.layers[params['LAYER']] else: return self.error( 400, "Wrong Layer '{}'".format(params['LAYER']), **kwargs) index = 3 dimensions = path[index:index + len(layer['dimensions'])] for dimension in layer['dimensions']: metadata["dimension_" + dimension['name']] = path[index] params[dimension['name'].upper()] = path[index] index += 1 last = path[-1].split('.') if len(path) < index + 4: # pragma: no cover return self.error(400, "Not enough path", **kwargs) params['TILEMATRIXSET'] = path[index] params['TILEMATRIX'] = path[index + 1] params['TILEROW'] = path[index + 2] if len(path) == index + 4: params['REQUEST'] = 'GetTile' params['TILECOL'] = last[0] if last[1] != layer['extension']: # pragma: no cover return self.error( 400, "Wrong extension '{}'".format(last[1]), **kwargs) elif len(path) == index + 6: params['REQUEST'] = 'GetFeatureInfo' params['TILECOL'] = path[index + 3] params['I'] = path[index + 4] params['J'] = last[0] params['INFO_FORMAT'] = layer.get( 'info_formats', ['application/vnd.ogc.gml'])[0] else: # pragma: no cover return self.error(400, "Wrong path length", **kwargs) params['FORMAT'] = layer['mime_type'] else: if \ 'SERVICE' not in params or \ 'REQUEST' not in params or \ 'VERSION' not in params: return self.error(400, "Not all required parameters are present", **kwargs) if params['SERVICE'] != 'WMTS': return self.error(400, "Wrong Service '{}'".format(params['SERVICE']), **kwargs) if params['VERSION'] != '1.0.0': return self.error(400, "Wrong Version '{}'".format(params['VERSION']), **kwargs) if params['REQUEST'] == 'GetCapabilities': wmtscapabilities_file = self.cache['wmtscapabilities_file'] body, mime = self._get(wmtscapabilities_file, **kwargs) if mime is not None: return self.response( body, headers={ 'Content-Type': "application/xml", 'Expires': (datetime.datetime.utcnow() + datetime.timedelta( hours=self.expires_hours)).isoformat(), 'Cache-Control': "max-age={}".format((3600 * self.expires_hours)), 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET', }, **kwargs) else: # pragma: no cover return body if \ 'FORMAT' not in params or \ 'LAYER' not in params or \ 'TILEMATRIXSET' not in params or \ 'TILEMATRIX' not in params or \ 'TILEROW' not in params or \ 'TILECOL' not in params: # pragma: no cover return self.error(400, "Not all required parameters are present", **kwargs) if not path: if params['LAYER'] in self.layers: layer = self.tilegeneration.layers[params['LAYER']] else: return self.error(400, "Wrong Layer '{}'".format(params['LAYER']), **kwargs) for dimension in layer['dimensions']: value = params[dimension['name'].upper()] \ if dimension['name'].upper() in params \ else dimension['default'] dimensions.append(value) metadata["dimension_" + dimension['name']] = value if params['STYLE'] != layer['wmts_style']: return self.error(400, "Wrong Style '{}'".format(params['STYLE']), **kwargs) if params['TILEMATRIXSET'] != layer['grid']: return self.error( 400, "Wrong TileMatrixSet '{}'".format(params['TILEMATRIXSET']), **kwargs) tile = Tile( TileCoord( # TODO fix for matrix_identifier = resolution int(params['TILEMATRIX']), int(params['TILECOL']), int(params['TILEROW']), ), metadata=metadata) if params['REQUEST'] == 'GetFeatureInfo': if \ 'I' not in params or \ 'J' not in params or \ 'INFO_FORMAT' not in params: # pragma: no cover return self.error(400, "Not all required parameters are present", **kwargs) if 'query_layers' in layer: return self.forward(layer['url'] + '?' + urlencode( { 'SERVICE': 'WMS', 'VERSION': layer['version'], 'REQUEST': 'GetFeatureInfo', 'LAYERS': layer['layers'], 'QUERY_LAYERS': layer['query_layers'], 'STYLES': params['STYLE'], 'FORMAT': params['FORMAT'], 'INFO_FORMAT': params['INFO_FORMAT'], 'WIDTH': layer['grid_ref']['tile_size'], 'HEIGHT': layer['grid_ref']['tile_size'], 'SRS': layer['grid_ref']['srs'], 'BBOX': layer['grid_ref']['obj'].extent( tile.tilecoord), 'X': params['I'], 'Y': params['J'], }), no_cache=True, **kwargs) else: # pragma: no cover return self.error( 400, "Layer '{}' not queryable".format(layer['name']), **kwargs) if params['REQUEST'] != 'GetTile': return self.error(400, "Wrong Request '{}'".format(params['REQUEST']), **kwargs) if params['FORMAT'] != layer['mime_type']: return self.error(400, "Wrong Format '{}'".format(params['FORMAT']), **kwargs) if tile.tilecoord.z > self.max_zoom_seed[ layer['name']]: # pragma: no cover return self.forward(self.mapcache_baseurl + '?' + urlencode(params), headers=self.mapcache_header, **kwargs) if layer['name'] in self.filters: layer_filter = self.filters[layer['name']] meta_size = layer['meta_size'] meta_tilecoord = TileCoord( # TODO fix for matrix_identifier = resolution tile.tilecoord.z, tile.tilecoord.x / meta_size * meta_size, tile.tilecoord.y / meta_size * meta_size, meta_size, ) if meta_size != 1 else tile.tilecoord if not layer_filter.filter_tilecoord( meta_tilecoord): # pragma: no cover return self.forward(self.mapcache_baseurl + '?' + urlencode(params), headers=self.mapcache_header, **kwargs) store_ref = '/'.join([params['LAYER']] + list(dimensions)) if store_ref in self.stores: # pragma: no cover store = self.stores[store_ref] else: # pragma: no cover return self.error( 400, "No store found for layer '{}' and dimensions {}".format( layer['name'], ', '.join(dimensions)), **kwargs) tile = store.get_one(tile) if tile: if tile.error: return self.error(500, tile.error, **kwargs) return self.response( tile.data, headers={ 'Content-Type': tile.content_type, 'Expires': (datetime.datetime.utcnow() + datetime.timedelta(hours=self.expires_hours)).isoformat(), 'Cache-Control': "max-age={}".format((3600 * self.expires_hours)), 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET', }, **kwargs) else: return self.error(204, **kwargs) def forward(self, url, headers=None, no_cache=False, **kwargs): if headers is None: headers = {} if no_cache: headers['Cache-Control'] = 'no-cache' headers['Pragma'] = 'no-cache' response = requests.get(url, headers=headers) if response.status_code == 200: response_headers = response.headers.copy() if no_cache: response_headers['Cache-Control'] = 'no-cache, no-store' response_headers['Pragma'] = 'no-cache' else: # pragma: no cover response_headers['Expires'] = ( datetime.datetime.utcnow() + datetime.timedelta(hours=self.expires_hours)).isoformat() response_headers['Cache-Control'] = "max-age={}".format( (3600 * self.expires_hours)) response_headers['Access-Control-Allow-Origin'] = '*' response_headers['Access-Control-Allow-Methods'] = 'GET' return self.response(response.content, headers=response_headers, **kwargs) else: # pragma: no cover message = "The URL '{}' return '{} {}', content:\n{}".format( url, response.status_code, response.reason, response.text) logger.warning(message) return self.error(502, message=message, **kwargs) HTTP_MESSAGES = { 204: '204 No Content', 400: '400 Bad Request', 403: '403 Forbidden', 404: '404 Not Found', 502: '502 Bad Gateway', } def error(self, code, message='', **kwargs): kwargs['start_response'](self.HTTP_MESSAGES[code], []) return [message] @staticmethod def response(data, headers=None, **kwargs): if headers is None: # pragma: no cover headers = {} headers['Content-Length'] = str(len(data)) kwargs['start_response']('200 OK', headers.items()) return [data]
def _copy( self, options: Namespace, gene: TileGeneration, layer_name: str, source: str, dest: str, task_name: str, ) -> None: # disable metatiles assert gene.config_file config = gene.get_config(gene.config_file) layer = config.config["layers"][layer_name] cast(tilecloud_chain.configuration.LayerWms, layer)["meta"] = False count_tiles_dropped = Count() gene.create_log_tiles_error(layer_name) source_tilestore = gene.get_tilesstore(source) dest_tilestore = gene.get_tilesstore(dest) gene.init_tilecoords(config, layer_name) gene.add_geom_filter() gene.add_logger() gene.get(source_tilestore, "Get the tiles") gene.imap(DropEmpty(gene)) # Discard tiles with certain content if "empty_tile_detection" in layer: empty_tile = layer["empty_tile_detection"] gene.imap( HashDropper(empty_tile["size"], empty_tile["hash"], store=dest_tilestore, count=count_tiles_dropped)) if options.process: gene.process(options.process) gene.imap(DropEmpty(gene)) self.count = gene.counter_size() gene.put(dest_tilestore, "Store the tiles") gene.consume() if not options.quiet: print(f"""The tile {task_name} of layer '{layer_name}' is finish Nb {task_name} tiles: {self.count.nb} Nb errored tiles: {gene.error} Nb dropped tiles: {count_tiles_dropped.nb} Total time: {duration_format(gene.duration)} Total size: {size_format(self.count.size)} Time per tile: {(gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0} ms Size per tile: {self.count.size / self.count.nb if self.count.nb != 0 else -1} o """)
def main() -> None: """Run the tiles generation.""" try: stats.init_backends({}) parser = ArgumentParser(description="Used to generate the tiles", prog=sys.argv[0]) add_common_options(parser, dimensions=True) parser.add_argument( "--get-hash", metavar="TILE", help="get the empty tiles hash, use the specified TILE z/x/y" ) parser.add_argument( "--get-bbox", metavar="TILE", help="get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles", ) parser.add_argument( "--role", default="local", choices=("local", "master", "slave"), help="local/master/slave, master to file the queue and slave to generate the tiles", ) parser.add_argument( "--local-process-number", default=None, help="The number of process that we run in parallel" ) parser.add_argument( "--detach", default=False, action="store_true", help="run detached from the terminal" ) parser.add_argument( "--daemon", default=False, action="store_true", help="run continuously as a daemon" ) parser.add_argument( "--tiles", metavar="FILE", help="Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles", ) options = parser.parse_args() if options.detach: detach() gene = TileGeneration( config_file=options.config, options=options, multi_thread=options.get_hash is None ) if ( options.get_hash is None and options.get_bbox is None and options.config is not None and "authorised_user" in gene.get_main_config().config.get("generation", {}) and gene.get_main_config().config["generation"]["authorised_user"] != getuser() ): logger.error( "not authorised, authorised user is: %s.", gene.get_main_config().config["generation"]["authorised_user"], ) sys.exit(1) if options.config: config = gene.get_config(options.config) if options.cache is None and options.config: options.cache = config.config["generation"]["default_cache"] if options.tiles is not None and options.role not in ["local", "master"]: logger.error("The --tiles option work only with role local or master") sys.exit(1) try: generate = Generate(options, gene) if options.role == "slave": generate.gene() elif options.layer: generate.gene(options.layer) elif options.get_bbox: logger.error("With --get-bbox option you need to specify a layer") sys.exit(1) elif options.get_hash: logger.error("With --get-hash option you need to specify a layer") sys.exit(1) else: if options.config: for layer in config.config["generation"].get( "default_layers", config.config["layers"].keys() ): generate.gene(layer) except tilecloud.filter.error.TooManyErrors: logger.exception("Too many errors") sys.exit(1) finally: gene.close() except SystemExit: raise except: # pylint: disable=bare-except logger.exception("Exit with exception") if os.environ.get("TESTS", "false").lower() == "true": raise sys.exit(1)
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog='./buildout/bin/generate_amazon') add_comon_options(parser) parser.add_argument('--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file') parser.add_argument('--status', default=False, action="store_true", help='display the SQS queue status and exit') parser.add_argument('--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation') parser.add_argument('--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code') parser.add_argument('--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database') parser.add_argument('--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling') parser.add_argument('--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation') parser.add_argument('--host', default=None, help='The host used to generate tiles') parser.add_argument('--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.') options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print "EC2 not configured" sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if options.geodata and 'geodata_folder' in gene.config['ec2']: print "==== Sync geodata ====" ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code: print "==== Sync and build code ====" error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True) if error: exit(1) # pragma: no cover cmd = [ 'rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run_remote(cmd, host, project_dir, gene) if 'apache_content' in gene.config[ 'ec2'] and 'apache_config' in gene.config['ec2']: run_remote( 'echo %s > %s' % (gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config']), host, project_dir, gene) run_remote('sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene)) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce(lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta()) / len(times)**2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print '==== Time results ====' print 'A tile is generated in: %0.3f [ms]' % mean_time_ms print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb) print '''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb) if options.shutdown: # pragma: no cover run_remote('sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue: # pragma: no cover print "==== Till queue ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master']) project_dir = gene.config['ec2']['code_folder'] run_remote( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene) if options.tiles_gen: # pragma: no cover print "==== Generate tiles ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'slave']) arguments.append("--daemonize") project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene)) if options.shutdown: for p in processes: p.communicate() # wait process end else: print 'Tile generation started in background' if options.shutdown: run_remote('sudo shutdown 0') if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller")
def main(): stats.init_backends({}) parser = ArgumentParser( description= 'Used to generate the contextual file like the capabilities, the legends, ' 'the Apache and MapCache configuration', prog=sys.argv[0]) add_comon_options(parser, tile_pyramid=False, no_geom=False) parser.add_argument('--status', default=False, action="store_true", help='Display the SQS queue status and exit') parser.add_argument('--capabilities', '--generate-wmts-capabilities', default=False, action='store_true', help='Generate the WMTS Capabilities') parser.add_argument('--legends', '--generate-legend-images', default=False, action='store_true', dest='legends', help='Generate the legend images') parser.add_argument('--openlayers', '--generate-openlayers-testpage', default=False, action='store_true', dest='openlayers', help='Generate openlayers test page') parser.add_argument('--mapcache', '--generate-mapcache-config', default=False, action='store_true', dest='mapcache', help='Generate MapCache configuration file') parser.add_argument('--apache', '--generate-apache-config', default=False, action='store_true', dest='apache', help='Generate Apache configuration file') parser.add_argument( '--dump-config', default=False, action='store_true', help='Dump the used config with default values and exit') options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(gene) sys.exit(0) if options.cache is None: options.cache = gene.config['generation']['default_cache'] if options.dump_config: for layer in gene.config['layers'].values(): gene.init_layer(layer, options) _validate_generate_wmts_capabilities(gene.caches[options.cache], True) for grid in gene.config['grids'].values(): if 'obj' in grid: del grid['obj'] print(yaml.dump(gene.config)) sys.exit(0) if options.legends: _generate_legend_images(gene) if options.capabilities: _generate_wmts_capabilities(gene) if options.mapcache: _generate_mapcache_config(gene) if options.apache: _generate_apache_config(gene) if options.openlayers: _generate_openlayers(gene)
def main(): parser = ArgumentParser( description="Used to generate the contextual file like the capabilities, the legends, " "the Apache and MapCache configuration", prog=sys.argv[0], ) add_comon_options(parser, tile_pyramid=False, no_geom=False) parser.add_argument( "--capabilities", "--generate-wmts-capabilities", default=False, action="store_true", help="Generate the WMTS Capabilities", ) parser.add_argument( "--legends", "--generate-legend-images", default=False, action="store_true", dest="legends", help="Generate the legend images", ) parser.add_argument( "--openlayers", "--generate-openlayers-test-page", default=False, action="store_true", dest="openlayers", help="Generate openlayers test page", ) parser.add_argument( "--mapcache", "--generate-mapcache-config", default=False, action="store_true", dest="mapcache", help="Generate MapCache configuration file", ) parser.add_argument( "--apache", "--generate-apache-config", default=False, action="store_true", dest="apache", help="Generate Apache configuration file", ) parser.add_argument( "--dump-config", default=False, action="store_true", help="Dump the used config with default values and exit" ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.cache is None: options.cache = gene.config["generation"]["default_cache"] if options.dump_config: for layer in gene.config["layers"].keys(): gene.set_layer(layer, options) validate_calculate_cost(gene) _validate_generate_wmts_capabilities(gene, gene.caches[options.cache]) gene.validate_mapcache_config() gene.validate_apache_config() _validate_generate_openlayers(gene) for grid in gene.config["grids"].values(): if "obj" in grid: del grid["obj"] print(yaml.dump(gene.config)) sys.exit(0) if options.legends: _generate_legend_images(gene) if options.capabilities: _generate_wmts_capabilities(gene) if options.mapcache: _generate_mapcache_config(gene) if options.apache: _generate_apache_config(gene) if options.openlayers: _generate_openlayers(gene)