Exemplo n.º 1
0
def main():
    parser = ArgumentParser(
        description="Used to copy the tiles from a cache to an other",
        prog=sys.argv[0])
    add_comon_options(parser,
                      near=False,
                      time=False,
                      dimensions=True,
                      cache=False)
    parser.add_argument("--process",
                        dest="process",
                        metavar="NAME",
                        help="The process name to do")
    parser.add_argument("source", metavar="SOURCE", help="The source cache")
    parser.add_argument("dest", metavar="DEST", help="The destination cache")

    options = parser.parse_args()

    gene = TileGeneration(options.config, options)

    if options.layer:  # pragma: no cover
        copy = Copy()
        copy.copy(options, gene, options.layer, options.source, options.dest,
                  "copy")
    else:
        layers = (gene.config["generation"]["default_layers"]
                  if "default_layers" in gene.config["generation"] else
                  gene.config["layers"].keys())
        for layer in layers:
            copy = Copy()
            copy.copy(options, gene, layer, options.source, options.dest,
                      "copy")
Exemplo n.º 2
0
def main():
    parser = ArgumentParser(
        description='Used to copy the tiles from a cache to an other',
        prog=sys.argv[0])
    add_comon_options(parser,
                      near=False,
                      time=False,
                      dimensions=True,
                      cache=False)
    parser.add_argument('--process',
                        dest='process',
                        metavar="NAME",
                        help='The process name to do')
    parser.add_argument('source', metavar="SOURCE", help='The source cache')
    parser.add_argument('dest', metavar="DEST", help='The destination cache')

    options = parser.parse_args()

    gene = TileGeneration(options.config, options)

    if options.layer:  # pragma: no cover
        copy = Copy()
        copy.copy(options, gene, options.layer, options.source, options.dest,
                  'copy')
    else:
        layers = gene.config['generation']['default_layers'] \
            if 'default_layers' in gene.config['generation'] \
            else gene.config['layers'].keys()
        for layer in layers:
            copy = Copy()
            copy.copy(options, gene, layer, options.source, options.dest,
                      'copy')
Exemplo n.º 3
0
def main():
    parser = ArgumentParser(
        description='Used to copy the tiles from a cache to an other', prog=sys.argv[0]
    )
    add_comon_options(parser, near=False, time=False, dimensions=True, cache=False)
    parser.add_argument(
        '--process', dest='process', metavar="NAME",
        help='The process name to do'
    )
    parser.add_argument(
        'source',
        metavar="SOURCE",
        help='The source cache'
    )
    parser.add_argument(
        'dest',
        metavar="DEST",
        help='The destination cache'
    )

    options = parser.parse_args()

    gene = TileGeneration(options.config, options)

    if (options.layer):  # pragma: no cover
        copy = Copy()
        copy.copy(options, gene, options.layer, options.source, options.dest, 'copy')
    else:
        layers = gene.config['generation']['default_layers'] \
            if 'default_layers' in gene.config['generation'] \
            else gene.config['layers'].keys()
        for layer in layers:
            copy = Copy()
            copy.copy(options, gene, layer, options.source, options.dest, 'copy')
Exemplo n.º 4
0
def main():
    parser = ArgumentParser(
        description='Used to calculate the generation cost',
        prog=sys.argv[0]
    )
    add_comon_options(parser, tile_pyramid=False)
    parser.add_argument(
        '--cost-algo', '--calculate-cost-algorithm', default='area', dest='cost_algo',
        choices=('area', 'count'),
        help="The algorithm use to calculate the cost default base on the 'area' "
        "of the generation geometry, can also be 'count', to be base on number of tiles to generate."
    )

    options = parser.parse_args()
    gene = TileGeneration(
        options.config, options,
        layer_name=options.layer, base_config={'cost': {}}
    )

    all_size = 0
    tile_size = 0
    all_tiles = 0
    if (options.layer):
        (all_size, all_time, all_price, all_tiles) = _calculate_cost(gene, options)
        tile_size = gene.layer['cost']['tile_size'] / (1024.0 * 1024)
    else:
        all_time = timedelta()
        all_price = 0
        for layer in gene.config['generation']['default_layers']:
            print("")
            print("===== %s =====" % layer)
            gene.set_layer(layer, options)
            (size, time, price, tiles) = _calculate_cost(gene, options)
            tile_size += gene.layer['cost']['tile_size'] / (1024.0 * 1024)
            all_time += time
            all_price += price
            all_size += size
            all_tiles += tiles

        print("")
        print("===== GLOBAL =====")
        print("Total number of tiles: %i" % all_tiles)
        print('Total generation time: %s [d h:mm:ss]' % (duration_format(all_time)))
        print('Total generation cost: %0.2f [$]' % all_price)
    print("")
    print('S3 Storage: %0.2f [$/month]' % (all_size * gene.config['cost']['s3']['storage'] / (1024.0 * 1024 * 1024)))
    print('S3 get: %0.2f [$/month]' % (
        gene.config['cost']['s3']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
        gene.config['cost']['s3']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
    )
#    if 'cloudfront' in gene.config['cost']:
#        print('CloudFront: %0.2f [$/month]' % ()
#            gene.config['cost']['cloudfront']['get'] * gene.config['cost']['request_per_layers'] / 10000.0 +
#            gene.config['cost']['cloudfront']['download'] * gene.config['cost']['request_per_layers'] * tile_size)
    if 'ec2' in gene.config:
        print('ESB storage: %0.2f [$/month]' % (
            gene.config['cost']['esb']['storage'] * gene.config['cost']['esb_size'])
        )
    sys.exit(0)
Exemplo n.º 5
0
def process():
    parser = ArgumentParser(
        description='Used to copy the tiles from a cache to an other', prog='./buildout/bin/generate_copy'
    )
    add_comon_options(parser, near=False, time=False, dimensions=True)
    parser.add_argument(
        'process', metavar="PROCESS",
        help='The process name to do'
    )

    options = parser.parse_args()

    gene = TileGeneration(options.config, options)

    if (options.layer):  # pragma: no cover
        copy = Copy()
        copy.copy(options, gene, options.layer, options.cache, options.cache, 'process')
    else:
        for layer in gene.config['generation']['default_layers']:
            copy = Copy()
            copy.copy(options, gene, layer, options.cache, options.cache, 'process')
Exemplo n.º 6
0
def process():
    parser = ArgumentParser(
        description='Used to copy the tiles from a cache to an other',
        prog='./buildout/bin/generate_copy')
    add_comon_options(parser, near=False, time=False, dimensions=True)
    parser.add_argument('process',
                        metavar="PROCESS",
                        help='The process name to do')

    options = parser.parse_args()

    gene = TileGeneration(options.config, options)

    if (options.layer):  # pragma: no cover
        copy = Copy()
        copy.copy(options, gene, options.layer, options.cache, options.cache,
                  'process')
    else:
        for layer in gene.config['generation']['default_layers']:
            copy = Copy()
            copy.copy(options, gene, layer, options.cache, options.cache,
                      'process')
Exemplo n.º 7
0
def process():
    parser = ArgumentParser(
        description='Used to copy the tiles from a cache to an other', prog=sys.argv[0]
    )
    add_comon_options(parser, near=False, time=False, dimensions=True)
    parser.add_argument(
        'process', metavar="PROCESS",
        help='The process name to do'
    )

    options = parser.parse_args()

    gene = TileGeneration(options.config, options)

    copy = Copy()
    if (options.layer):  # pragma: no cover
        copy.copy(options, gene, options.layer, options.cache, options.cache, 'process')
    else:
        layers_name = gene.config['generation']['default_layers'] \
            if 'default_layers' in gene.config.get('generation', {}) \
            else gene.layers.keys()
        for layer in layers_name:
            copy.copy(options, gene, layer, options.cache, options.cache, 'process')
Exemplo n.º 8
0
def process():
    parser = ArgumentParser(
        description='Used to copy the tiles from a cache to an other',
        prog=sys.argv[0])
    add_comon_options(parser, near=False, time=False, dimensions=True)
    parser.add_argument('process',
                        metavar="PROCESS",
                        help='The process name to do')

    options = parser.parse_args()

    gene = TileGeneration(options.config, options)

    copy = Copy()
    if options.layer:  # pragma: no cover
        copy.copy(options, gene, options.layer, options.cache, options.cache,
                  'process')
    else:
        layers_name = gene.config['generation']['default_layers'] \
            if 'default_layers' in gene.config.get('generation', {}) \
            else gene.layers.keys()
        for layer in layers_name:
            copy.copy(options, gene, layer, options.cache, options.cache,
                      'process')
Exemplo n.º 9
0
def process():
    parser = ArgumentParser(
        description="Used to copy the tiles from a cache to an other",
        prog=sys.argv[0])
    add_comon_options(parser, near=False, time=False, dimensions=True)
    parser.add_argument("process",
                        metavar="PROCESS",
                        help="The process name to do")

    options = parser.parse_args()

    gene = TileGeneration(options.config, options, multi_thread=False)

    copy = Copy()
    if options.layer:  # pragma: no cover
        copy.copy(options, gene, options.layer, options.cache, options.cache,
                  "process")
    else:
        layers_name = (gene.config["generation"]["default_layers"]
                       if "default_layers" in gene.config.get(
                           "generation", {}) else gene.layers.keys())
        for layer in layers_name:
            copy.copy(options, gene, layer, options.cache, options.cache,
                      "process")
Exemplo n.º 10
0
def main():
    parser = ArgumentParser(
        description='Used to generate the tiles from Amazon EC2, '
        'and get the SQS queue status',
        prog='./buildout/bin/generate_amazon')
    add_comon_options(parser)
    parser.add_argument('--deploy-config',
                        default=None,
                        dest="deploy_config",
                        metavar="FILE",
                        help='path to the deploy configuration file')
    parser.add_argument('--status',
                        default=False,
                        action="store_true",
                        help='display the SQS queue status and exit')
    parser.add_argument('--disable-geodata',
                        default=True,
                        action="store_false",
                        dest="geodata",
                        help='disable geodata synchronisation')
    parser.add_argument('--disable-code',
                        default=True,
                        action="store_false",
                        dest="deploy_code",
                        help='disable deploy application code')
    parser.add_argument('--disable-database',
                        default=True,
                        action="store_false",
                        dest="deploy_database",
                        help='disable deploy database')
    parser.add_argument('--disable-fillqueue',
                        default=True,
                        action="store_false",
                        dest="fill_queue",
                        help='disable queue filling')
    parser.add_argument('--disable-tilesgen',
                        default=True,
                        action="store_false",
                        dest="tiles_gen",
                        help='disable tile generation')
    parser.add_argument('--host',
                        default=None,
                        help='The host used to generate tiles')
    parser.add_argument('--shutdown',
                        default=False,
                        action="store_true",
                        help='Shut done the remote host after the task.')

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.status:  # pragma: no cover
        status(options, gene)
        sys.exit(0)

    if 'ec2' not in gene.config:  # pragma: no cover
        print "EC2 not configured"
        sys.exit(1)

    if options.deploy_config is None:
        options.deploy_config = gene.config['ec2']['deploy_config']
    if options.geodata:
        options.geodata = not gene.config['ec2']['disable_geodata']
    if options.deploy_code:
        options.deploy_code = not gene.config['ec2']['disable_code']
    if options.deploy_database:
        options.deploy_database = not gene.config['ec2']['disable_database']
    if options.fill_queue:  # pragma: no cover
        options.fill_queue = not gene.config['ec2']['disable_fillqueue']
    if options.tiles_gen:  # pragma: no cover
        options.tiles_gen = not gene.config['ec2']['disable_tilesgen']

    # start aws
    if not options.host:
        # TODO not implemented yet
        host = aws_start(gene.config['ec2']['host_type'])  # pragma: no cover
    else:
        host = options.host

    if options.geodata and 'geodata_folder' in gene.config['ec2']:
        print "==== Sync geodata ===="
        ssh_options = ''
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            ssh_options = gene.config['ec2']['ssh_options']
        # sync geodata
        run_local([
            'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
            gene.config['ec2']['geodata_folder'],
            host + ':' + gene.config['ec2']['geodata_folder']
        ])

    if options.deploy_code:
        print "==== Sync and build code ===="
        error = gene.validate(gene.config['ec2'],
                              'ec2',
                              'code_folder',
                              required=True)
        if error:
            exit(1)  # pragma: no cover

        cmd = [
            'rsync',
            '--delete',
        ]
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
            ssh_options = gene.config['ec2']['ssh_options']

        project_dir = gene.config['ec2']['code_folder']
        cmd += ['-r', '.', host + ':' + project_dir]
        run_local(cmd)

        for cmd in gene.config['ec2']['build_cmds']:
            run_remote(cmd, host, project_dir, gene)
        if 'apache_content' in gene.config[
                'ec2'] and 'apache_config' in gene.config['ec2']:
            run_remote(
                'echo %s > %s' % (gene.config['ec2']['apache_content'],
                                  gene.config['ec2']['apache_config']), host,
                project_dir, gene)
        run_remote('sudo apache2ctl graceful', host, project_dir, gene)

    # deploy
    if options.deploy_database:
        _deploy(gene, host)

    if options.deploy_code or options.deploy_database \
            or options.geodata:
        # TODO not implemented yet
        create_snapshot(host, gene)

    if options.time:
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'local'])
        arguments.extend(['--time', str(options.time)])

        project_dir = gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    './buildout/bin/generate_tiles ' +
                    ' '.join([str(a) for a in arguments]), host, project_dir,
                    gene))

        tiles_size = []
        times = []
        for p in processes:
            results = p.communicate()
            if results[1] != '':  # pragma: no cover
                logger.debug('ERROR: %s' % results[1])
            results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
            results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
            for r in results[0].split('\n'):
                if r.startswith('time: '):
                    times.append(int(r.replace('time: ', '')))
                elif r.startswith('size: '):
                    tiles_size.append(int(r.replace('size: ', '')))

        if len(times) == 0:  # pragma: no cover
            logger.error("Not enough data")
            sys.exit(1)
        mean_time = reduce(lambda x, y: x + y,
                           [timedelta(microseconds=int(r))
                            for r in times], timedelta()) / len(times)**2
        mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0

        mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
                           0) / len(tiles_size)
        mean_size_kb = mean_size / 1024.0

        print '==== Time results ===='
        print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
        print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
        print '''config:
    cost:
        tileonly_generation_time: %0.3f
        tile_generation_time: %0.3f
        metatile_generation_time: 0
        tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)

        if options.shutdown:  # pragma: no cover
            run_remote('sudo shutdown 0', host, project_dir, gene)
        sys.exit(0)

    if options.fill_queue:  # pragma: no cover
        print "==== Till queue ===="
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'master'])

        project_dir = gene.config['ec2']['code_folder']
        run_remote(
            './buildout/bin/generate_tiles ' +
            ' '.join([str(a) for a in arguments]), host, project_dir, gene)

    if options.tiles_gen:  # pragma: no cover
        print "==== Generate tiles ===="
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'slave'])
        arguments.append("--daemonize")

        project_dir = gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    './buildout/bin/generate_tiles ' +
                    ' '.join([str(a) for a in arguments]), host, project_dir,
                    gene))

        if options.shutdown:
            for p in processes:
                p.communicate()  # wait process end
        else:
            print 'Tile generation started in background'

        if options.shutdown:
            run_remote('sudo shutdown 0')

        if 'sns' in gene.config:
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(
                    gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
                    'host': socket.getfqdn(),
                    'cmd': ' '.join([quote(arg) for arg in sys.argv])
                }, "Tile generation controller")
Exemplo n.º 11
0
def main():
    stats.init_backends({})
    parser = ArgumentParser(
        description="Used to generate the contextual file like the capabilities, the legends, "
        "the Apache and MapCache configuration",
        prog=sys.argv[0],
    )
    add_comon_options(parser, tile_pyramid=False, no_geom=False)
    parser.add_argument(
        "--status", default=False, action="store_true", help="Display the SQS queue status and exit"
    )
    parser.add_argument(
        "--capabilities",
        "--generate-wmts-capabilities",
        default=False,
        action="store_true",
        help="Generate the WMTS Capabilities",
    )
    parser.add_argument(
        "--legends",
        "--generate-legend-images",
        default=False,
        action="store_true",
        dest="legends",
        help="Generate the legend images",
    )
    parser.add_argument(
        "--openlayers",
        "--generate-openlayers-testpage",
        default=False,
        action="store_true",
        dest="openlayers",
        help="Generate openlayers test page",
    )
    parser.add_argument(
        "--mapcache",
        "--generate-mapcache-config",
        default=False,
        action="store_true",
        dest="mapcache",
        help="Generate MapCache configuration file",
    )
    parser.add_argument(
        "--mapcache-version", default="1.4", choices=("1.4", "1.6"), help="The used version of MapCache"
    )
    parser.add_argument(
        "--apache",
        "--generate-apache-config",
        default=False,
        action="store_true",
        dest="apache",
        help="Generate Apache configuration file",
    )
    parser.add_argument(
        "--dump-config",
        default=False,
        action="store_true",
        help="Dump the used config with default values and exit",
    )

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.status:  # pragma: no cover
        status(gene)
        sys.exit(0)

    if options.cache is None:
        options.cache = gene.config["generation"]["default_cache"]

    if options.dump_config:
        for layer in gene.config["layers"].values():
            gene.init_layer(layer, options)
        _validate_generate_wmts_capabilities(gene.caches[options.cache], True)
        for grid in gene.config["grids"].values():
            if "obj" in grid:
                del grid["obj"]
        print(yaml.dump(gene.config))
        sys.exit(0)

    if options.legends:
        _generate_legend_images(gene)

    if options.capabilities:
        _generate_wmts_capabilities(gene)

    if options.mapcache:
        _generate_mapcache_config(gene, options.mapcache_version)

    if options.apache:
        _generate_apache_config(gene)

    if options.openlayers:
        _generate_openlayers(gene)
Exemplo n.º 12
0
def main():
    parser = ArgumentParser(description='Used to generate the tiles',
                            prog='./buildout/bin/generate_tiles')
    add_comon_options(parser, dimensions=True)
    parser.add_argument(
        '--get-hash',
        metavar="TILE",
        help='get the empty tiles hash, use the specified TILE z/x/y')
    parser.add_argument(
        '--get-bbox',
        metavar="TILE",
        help=
        'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
    )
    parser.add_argument(
        '--role',
        default='local',
        choices=('local', 'master', 'slave'),
        help='local/master/slave, master to file the queue and '
        'slave to generate the tiles')
    parser.add_argument('--daemonize',
                        default=False,
                        action="store_true",
                        help='run as a daemon')
    parser.add_argument(
        '--tiles',
        metavar="FILE",
        help=
        'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
    )
    parser.add_argument('--generated-tiles-file',
                        metavar="FILE",
                        help='Store the tiles in a file (unrecommended)')

    options = parser.parse_args()

    if options.daemonize:
        daemonize()  # pragma: no cover

    gene = TileGeneration(options.config, options)

    if options.get_hash is None and options.get_bbox is None and \
            'authorised_user' in gene.config['generation'] and \
            gene.config['generation']['authorised_user'] != getuser():
        exit('not authorised, authorised user is: %s.' %
             gene.config['generation']['authorised_user'])

    if options.cache is None:
        options.cache = gene.config['generation']['default_cache']

    if options.tiles is not None and options.role not in [
            'local', 'master'
    ]:  # pragma: no cover
        exit("The --tiles option work only with role local or master")

    try:
        if (options.layer):
            generate = Generate()
            generate.gene(options, gene, options.layer)
        elif options.get_bbox:  # pragma: no cover
            exit("With --get-bbox option we needs to specify a layer")
        elif options.get_hash:  # pragma: no cover
            exit("With --get-hash option we needs to specify a layer")
        elif options.tiles:  # pragma: no cover
            exit("With --tiles option we needs to specify a layer")
        else:
            for layer in gene.config['generation']['default_layers']:
                generate = Generate()
                generate.gene(options, gene, layer)
    finally:
        if gene.error_file is not None:
            gene.error_file.close()
Exemplo n.º 13
0
def main():
    parser = ArgumentParser(
        description='Used to generate the tiles from Amazon EC2, '
        'and get the SQS queue status',
        prog='./buildout/bin/generate_amazon'
    )
    add_comon_options(parser)
    parser.add_argument(
        '--deploy-config', default=None, dest="deploy_config", metavar="FILE",
        help='path to the deploy configuration file'
    )
    parser.add_argument(
        '--status', default=False, action="store_true",
        help='display the SQS queue status and exit'
    )
    parser.add_argument(
        '--disable-geodata', default=True, action="store_false", dest="geodata",
        help='disable geodata synchronisation'
    )
    parser.add_argument(
        '--disable-code', default=True, action="store_false", dest="deploy_code",
        help='disable deploy application code'
    )
    parser.add_argument(
        '--disable-database', default=True, action="store_false", dest="deploy_database",
        help='disable deploy database'
    )
    parser.add_argument(
        '--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
        help='disable queue filling'
    )
    parser.add_argument(
        '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
        help='disable tile generation'
    )
    parser.add_argument(
        '--host', default=None,
        help='The host used to generate tiles'
    )
    parser.add_argument(
        '--shutdown', default=False, action="store_true",
        help='Shut done the remote host after the task.'
    )

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.status:  # pragma: no cover
        status(options, gene)
        sys.exit(0)

    if 'ec2' not in gene.config:  # pragma: no cover
        print "EC2 not configured"
        sys.exit(1)

    if options.deploy_config is None:
        options.deploy_config = gene.config['ec2']['deploy_config']
    if options.geodata:
        options.geodata = not gene.config['ec2']['disable_geodata']
    if options.deploy_code:
        options.deploy_code = not gene.config['ec2']['disable_code']
    if options.deploy_database:
        options.deploy_database = not gene.config['ec2']['disable_database']
    if options.fill_queue:  # pragma: no cover
        options.fill_queue = not gene.config['ec2']['disable_fillqueue']
    if options.tiles_gen:  # pragma: no cover
        options.tiles_gen = not gene.config['ec2']['disable_tilesgen']

    # start aws
    if not options.host:
        # TODO not implemented yet
        host = aws_start(gene.config['ec2']['host_type'])  # pragma: no cover
    else:
        host = options.host

    if options.geodata and 'geodata_folder' in gene.config['ec2']:
        print "==== Sync geodata ===="
        ssh_options = ''
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            ssh_options = gene.config['ec2']['ssh_options']
        # sync geodata
        run_local([
            'rsync', '--delete', '-e', 'ssh ' + ssh_options,
            '-r', gene.config['ec2']['geodata_folder'],
            host + ':' + gene.config['ec2']['geodata_folder']
        ])

    if options.deploy_code:
        print "==== Sync and build code ===="
        error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
        if error:
            exit(1)  # pragma: no cover

        cmd = ['rsync', '--delete', ]
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
            ssh_options = gene.config['ec2']['ssh_options']

        project_dir = gene.config['ec2']['code_folder']
        cmd += ['-r', '.', host + ':' + project_dir]
        run_local(cmd)

        for cmd in gene.config['ec2']['build_cmds']:
            run_remote(cmd, host, project_dir, gene)
        if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
            run_remote(
                'echo %s > %s' % (
                    gene.config['ec2']['apache_content'],
                    gene.config['ec2']['apache_config']
                ), host, project_dir, gene
            )
        run_remote('sudo apache2ctl graceful', host, project_dir, gene)

    # deploy
    if options.deploy_database:
        _deploy(gene, host)

    if options.deploy_code or options.deploy_database \
            or options.geodata:
        # TODO not implemented yet
        create_snapshot(host, gene)

    if options.time:
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'local'])
        arguments.extend(['--time', str(options.time)])

        project_dir = gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    './buildout/bin/generate_tiles ' +
                    ' '.join([str(a) for a in arguments]), host, project_dir, gene
                )
            )

        tiles_size = []
        times = []
        for p in processes:
            results = p.communicate()
            if results[1] != '':  # pragma: no cover
                logger.debug('ERROR: %s' % results[1])
            results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
            results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
            for r in results[0].split('\n'):
                if r.startswith('time: '):
                    times.append(int(r.replace('time: ', '')))
                elif r.startswith('size: '):
                    tiles_size.append(int(r.replace('size: ', '')))

        if len(times) == 0:  # pragma: no cover
            logger.error("Not enough data")
            sys.exit(1)
        mean_time = reduce(
            lambda x, y: x + y,
            [timedelta(microseconds=int(r)) for r in times],
            timedelta()
        ) / len(times) ** 2
        mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0

        mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
        mean_size_kb = mean_size / 1024.0

        print '==== Time results ===='
        print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
        print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
        print '''config:
    cost:
        tileonly_generation_time: %0.3f
        tile_generation_time: %0.3f
        metatile_generation_time: 0
        tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)

        if options.shutdown:  # pragma: no cover
            run_remote('sudo shutdown 0', host, project_dir, gene)
        sys.exit(0)

    if options.fill_queue:  # pragma: no cover
        print "==== Till queue ===="
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'master'])

        project_dir = gene.config['ec2']['code_folder']
        run_remote(
            './buildout/bin/generate_tiles ' +
            ' '.join([str(a) for a in arguments]), host, project_dir, gene
        )

    if options.tiles_gen:  # pragma: no cover
        print "==== Generate tiles ===="
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'slave'])
        arguments.append("--daemonize")

        project_dir = gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    './buildout/bin/generate_tiles ' +
                    ' '.join([str(a) for a in arguments]), host, project_dir, gene)
            )

        if options.shutdown:
            for p in processes:
                p.communicate()  # wait process end
        else:
            print 'Tile generation started in background'

        if options.shutdown:
            run_remote('sudo shutdown 0')

        if 'sns' in gene.config:
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'],
                """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
                {
                    'host': socket.getfqdn(),
                    'cmd': ' '.join([quote(arg) for arg in sys.argv])
                },
                "Tile generation controller")
Exemplo n.º 14
0
def main():
    parser = ArgumentParser(
        description='Used to generate the contextual file like the capabilities, the legends, '
        'the Apache and MapCache configuration',
        prog='./buildout/bin/generate_controller'
    )
    add_comon_options(parser, tile_pyramid=False, no_geom=False)
    parser.add_argument(
        '--capabilities', '--generate_wmts_capabilities', default=False, action="store_true",
        help='Generate the WMTS Capabilities'
    )
    parser.add_argument(
        '--legends', '--generate_legend_images', default=False, action="store_true", dest='legends',
        help='Generate the legend images'
    )
    parser.add_argument(
        '--openlayers', '--generate-openlayers-test-page', default=False,
        action="store_true", dest='openlayers',
        help='Generate openlayers test page'
    )
    parser.add_argument(
        '--mapcache', '--generate-mapcache-config', default=False, action="store_true", dest='mapcache',
        help='Generate MapCache configuration file'
    )
    parser.add_argument(
        '--apache', '--generate-apache-config', default=False, action="store_true", dest='apache',
        help='Generate Apache configuration file'
    )
    parser.add_argument(
        '--dump-config', default=False, action="store_true",
        help='Dump the used config with default values and exit'
    )

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.cache is None:
        options.cache = gene.config['generation']['default_cache']

    if options.dump_config:
        for layer in gene.config['layers'].keys():
            gene.set_layer(layer, options)
            validate_calculate_cost(gene)
        _validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
        gene.validate_mapcache_config()
        gene.validate_apache_config()
        _validate_generate_openlayers(gene)
        for grid in gene.config['grids'].values():
            if 'obj' in grid:
                del grid['obj']
        print yaml.dump(gene.config)
        sys.exit(0)

    if options.legends:
        _generate_legend_images(gene)

    if options.capabilities:
        _generate_wmts_capabilities(gene)

    if options.mapcache:
        _generate_mapcache_config(gene)

    if options.apache:
        _generate_apache_config(gene)

    if options.openlayers:
        _generate_openlayers(gene)
Exemplo n.º 15
0
def main():
    parser = ArgumentParser(
        description='Used to generate the tiles from Amazon EC2, '
        'and get the SQS queue status',
        prog=sys.argv[0]
    )
    add_comon_options(parser)
    parser.add_argument(
        '--deploy-config', default=None, dest="deploy_config", metavar="FILE",
        help='path to the deploy configuration file'
    )
    parser.add_argument(
        '--status', default=False, action="store_true",
        help='display the SQS queue status and exit'
    )
    parser.add_argument(
        '--disable-geodata', default=True, action="store_false", dest="geodata",
        help='disable geodata synchronisation'
    )
    parser.add_argument(
        '--disable-code', default=True, action="store_false", dest="deploy_code",
        help='disable deploy application code'
    )
    parser.add_argument(
        '--disable-database', default=True, action="store_false", dest="deploy_database",
        help='disable deploy database'
    )
    parser.add_argument(
        '--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
        help='disable queue filling'
    )
    parser.add_argument(
        '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
        help='disable tile generation'
    )
    parser.add_argument(
        '--host', default=None,
        help='The host used to generate tiles'
    )
    parser.add_argument(
        '--shutdown', default=False, action="store_true",
        help='Shut done the remote host after the task.'
    )
    parser.add_argument(
        '--wait', default=False, action="store_true",
        help='Wait that all the tasks will finish.'
    )
    parser.add_argument(
        '--local', default=False, action="store_true",
        help='Run the generation locally'
    )

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.status:  # pragma: no cover
        status(options, gene)
        sys.exit(0)

    if 'ec2' not in gene.config:  # pragma: no cover
        print("EC2 not configured")
        sys.exit(1)

    if options.deploy_config is None:
        options.deploy_config = gene.config['ec2']['deploy_config']
    if options.geodata:
        options.geodata = not gene.config['ec2']['disable_geodata']
    if options.deploy_code:
        options.deploy_code = not gene.config['ec2']['disable_code']
    if options.deploy_database:
        options.deploy_database = not gene.config['ec2']['disable_database']
    if options.fill_queue:  # pragma: no cover
        options.fill_queue = not gene.config['ec2']['disable_fillqueue']
    if options.tiles_gen:  # pragma: no cover
        options.tiles_gen = not gene.config['ec2']['disable_tilesgen']

    # start aws
    if not options.host:
        # TODO not implemented yet
        host = aws_start(gene.config['ec2']['host_type'])  # pragma: no cover
    else:
        host = options.host

    if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']:
        print("==== Sync geodata ====")
        ssh_options = ''
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            ssh_options = gene.config['ec2']['ssh_options']
        # sync geodata
        run_local([
            'rsync', '--delete', '-e', 'ssh ' + ssh_options,
            '-r', gene.config['ec2']['geodata_folder'],
            host + ':' + gene.config['ec2']['geodata_folder']
        ])

    if options.deploy_code and not options.local:
        print("==== Sync and build code ====")
        error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
        if error:
            exit(1)  # pragma: no cover

        cmd = ['rsync', '--delete', ]
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
            ssh_options = gene.config['ec2']['ssh_options']

        project_dir = gene.config['ec2']['code_folder']
        cmd += ['-r', '.', host + ':' + project_dir]
        run_local(cmd)

        for cmd in gene.config['ec2']['build_cmds']:
            run(options, cmd % environ, host, project_dir, gene)
        if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
            run(
                options,
                'echo %s > %s' % (
                    gene.config['ec2']['apache_content'],
                    gene.config['ec2']['apache_config']
                ), host, project_dir, gene
            )
        run(options, 'sudo apache2ctl graceful', host, project_dir, gene)

    # deploy
    if options.deploy_database and not options.local:
        _deploy(gene, host)

    if options.deploy_code or options.deploy_database \
            or options.geodata and not options.local:
        # TODO not implemented yet
        create_snapshot(host, gene)

    if options.time:
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'local'])
        arguments.extend(['--time', str(options.time)])

        project_dir = None if options.local else gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    "%sgenerate_tiles %s" % (
                        _get_path(),
                        ' '.join([str(a) for a in arguments])
                    ), host, project_dir, gene
                )
            )

        tiles_size = []
        times = []
        for p in processes:
            results = p.communicate()
            if results[1] != '':  # pragma: no cover
                logger.debug('ERROR: %s' % results[1])
            if PY3:
                results = [r.decode('utf-8') for r in results]
            results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
            results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
            for r in results[0].split('\n'):
                if r.startswith('time: '):
                    times.append(int(r.replace('time: ', '')))
                elif r.startswith('size: '):
                    tiles_size.append(int(r.replace('size: ', '')))

        if len(times) == 0:  # pragma: no cover
            logger.error("Not enough data")
            sys.exit(1)
        mean_time = reduce(
            lambda x, y: x + y,
            [timedelta(microseconds=int(r)) for r in times],
            timedelta()
        ) / len(times) ** 2
        mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0

        mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
        mean_size_kb = mean_size / 1024.0

        print('==== Time results ====')
        print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
        print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
        print('''config:
    cost:
        tileonly_generation_time: %0.3f
        tile_generation_time: %0.3f
        metatile_generation_time: 0
        tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))

        if options.shutdown:  # pragma: no cover
            run(options, 'sudo shutdown 0', host, project_dir, gene)
        sys.exit(0)

    if options.fill_queue and not options.local:  # pragma: no cover
        print("==== Till queue ====")
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'master', '--quiet'])

        project_dir = gene.config['ec2']['code_folder']
        run_remote_process(
            options,
            "%sgenerate_tiles %s" % (
                _get_path(),
                ' '.join([str(a) for a in arguments])
            ), host, project_dir, gene
        )
        sleep(5)
        attributes = gene.get_sqs_queue().get_attributes()
        print(
            "\rTiles to generate: %s/%s" % (
                attributes['ApproximateNumberOfMessages'],
                attributes['ApproximateNumberOfMessagesNotVisible'],
            )
        )

    if options.tiles_gen:  # pragma: no cover
        print("==== Generate tiles ====")

        if options.wait and not options.local:
            print("")

            class Status(Thread):
                def run(self):  # pragma: no cover
                    while True:
                        attributes = gene.get_sqs_queue().get_attributes()
                        print(
                            "\rTiles to generate/generating: %s/%s" % (
                                attributes['ApproximateNumberOfMessages'],
                                attributes['ApproximateNumberOfMessagesNotVisible'],
                            )
                        )

                        sleep(1)
            status_thread = Status()
            status_thread.setDaemon(True)
            status_thread.start()

        arguments = _get_arguments(options)
        arguments.extend(['--quiet'])
        if not options.local:
            arguments.extend(['--role', 'slave'])

        project_dir = None if options.local else gene.config['ec2']['code_folder']
        threads = []
        for i in range(gene.config['ec2']['number_process']):
            if options.local:
                threads.append(run_local_process(
                    "%sgenerate_tiles --local-process-number %i %s" % (
                        _get_path(),
                        i, ' '.join([str(a) for a in arguments])
                    )
                ))
            else:
                run_remote_process(
                    "%sgenerate_tiles %s" % (
                        _get_path(),
                        ' '.join([str(a) for a in arguments])
                    ), host, project_dir, gene
                )

        print('Tile generation started')

        if options.shutdown:
            run(options, 'sudo shutdown 0')

        if options.wait and options.local:
            while len(threads) > 0:
                threads = [t for t in threads if t.is_alive()]
                sleep(1)

        if 'sns' in gene.config:
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'],
                """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
                {
                    'host': socket.getfqdn(),
                    'cmd': ' '.join([quote(arg) for arg in sys.argv])
                },
                "Tile generation controller"
            )
Exemplo n.º 16
0
def main():
    parser = ArgumentParser(
        description=
        'Used to generate the contextual file like the capabilities, the legends, '
        'the Apache and MapCache configuration',
        prog='./buildout/bin/generate_controller')
    add_comon_options(parser, tile_pyramid=False, no_geom=False)
    parser.add_argument('--capabilities',
                        '--generate_wmts_capabilities',
                        default=False,
                        action="store_true",
                        help='Generate the WMTS Capabilities')
    parser.add_argument('--legends',
                        '--generate_legend_images',
                        default=False,
                        action="store_true",
                        dest='legends',
                        help='Generate the legend images')
    parser.add_argument('--openlayers',
                        '--generate-openlayers-test-page',
                        default=False,
                        action="store_true",
                        dest='openlayers',
                        help='Generate openlayers test page')
    parser.add_argument('--mapcache',
                        '--generate-mapcache-config',
                        default=False,
                        action="store_true",
                        dest='mapcache',
                        help='Generate MapCache configuration file')
    parser.add_argument('--apache',
                        '--generate-apache-config',
                        default=False,
                        action="store_true",
                        dest='apache',
                        help='Generate Apache configuration file')
    parser.add_argument(
        '--dump-config',
        default=False,
        action="store_true",
        help='Dump the used config with default values and exit')

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.cache is None:
        options.cache = gene.config['generation']['default_cache']

    if options.dump_config:
        for layer in gene.config['layers'].keys():
            gene.set_layer(layer, options)
            validate_calculate_cost(gene)
        _validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
        gene.validate_mapcache_config()
        gene.validate_apache_config()
        _validate_generate_openlayers(gene)
        for grid in gene.config['grids'].values():
            if 'obj' in grid:
                del grid['obj']
        print yaml.dump(gene.config)
        sys.exit(0)

    if options.legends:
        _generate_legend_images(gene)

    if options.capabilities:
        _generate_wmts_capabilities(gene)

    if options.mapcache:
        _generate_mapcache_config(gene)

    if options.apache:
        _generate_apache_config(gene)

    if options.openlayers:
        _generate_openlayers(gene)
Exemplo n.º 17
0
def main():
    parser = ArgumentParser(description='Used to generate the tiles', prog=sys.argv[0])
    add_comon_options(parser, dimensions=True)
    parser.add_argument(
        '--get-hash', metavar="TILE",
        help='get the empty tiles hash, use the specified TILE z/x/y'
    )
    parser.add_argument(
        '--get-bbox', metavar="TILE",
        help='get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
    )
    parser.add_argument(
        '--role', default='local', choices=('local', 'master', 'slave'),
        help='local/master/slave, master to file the queue and '
        'slave to generate the tiles'
    )
    parser.add_argument(
        "--local-process-number", default=None,
        help="The number of process that we run in parallel"
    )
    parser.add_argument(
        '--daemonize', default=False, action="store_true",
        help='run as a daemon'
    )
    parser.add_argument(
        '--tiles', metavar="FILE",
        help='Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
    )
    parser.add_argument(
        '--generated-tiles-file', metavar="FILE",
        help='Store the tiles in a file (unrecommended)'
    )

    options = parser.parse_args()

    if options.daemonize:
        daemonize()  # pragma: no cover

    gene = TileGeneration(options.config, options)

    if options.get_hash is None and options.get_bbox is None and \
            'authorised_user' in gene.config['generation'] and \
            gene.config['generation']['authorised_user'] != getuser():
        exit('not authorised, authorised user is: %s.' % gene.config['generation']['authorised_user'])

    if options.cache is None:
        options.cache = gene.config['generation']['default_cache']

    if options.tiles is not None and options.role not in ['local', 'master']:  # pragma: no cover
        exit("The --tiles option work only with role local or master")

    try:
        if (options.layer):
            generate = Generate()
            generate.gene(options, gene, options.layer)
        elif options.get_bbox:  # pragma: no cover
            exit("With --get-bbox option we needs to specify a layer")
        elif options.get_hash:  # pragma: no cover
            exit("With --get-hash option we needs to specify a layer")
        elif options.tiles:  # pragma: no cover
            exit("With --tiles option we needs to specify a layer")
        else:
            for layer in gene.config['generation'].get('default_layers', gene.layers.keys()):
                generate = Generate()
                generate.gene(options, gene, layer)
    finally:
        if gene.error_file is not None:
            gene.error_file.close()
Exemplo n.º 18
0
def main():
    parser = ArgumentParser(description='Used to generate the tiles',
                            prog=sys.argv[0])
    add_comon_options(parser, dimensions=True)
    parser.add_argument(
        '--get-hash',
        metavar="TILE",
        help='get the empty tiles hash, use the specified TILE z/x/y')
    parser.add_argument(
        '--get-bbox',
        metavar="TILE",
        help=
        'get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
    )
    parser.add_argument(
        '--role',
        default='local',
        choices=('local', 'master', 'slave'),
        help='local/master/slave, master to file the queue and '
        'slave to generate the tiles')
    parser.add_argument("--local-process-number",
                        default=None,
                        help="The number of process that we run in parallel")
    parser.add_argument('--detach',
                        default=False,
                        action="store_true",
                        help='run detached from the terminal')
    parser.add_argument('--daemon',
                        default=False,
                        action="store_true",
                        help='run continuously as a daemon')
    parser.add_argument(
        '--tiles',
        metavar="FILE",
        help=
        'Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles'
    )
    parser.add_argument('--generated-tiles-file',
                        metavar="FILE",
                        help='Store the tiles in a file (unrecommended)')

    options = parser.parse_args()

    if options.detach:
        detach()  # pragma: no cover

    gene = TileGeneration(options.config, options)

    if options.get_hash is None and options.get_bbox is None and \
            'authorised_user' in gene.config['generation'] and \
            gene.config['generation']['authorised_user'] != getuser():
        exit('not authorised, authorised user is: {}.'.format(
            gene.config['generation']['authorised_user']))

    if options.cache is None:
        options.cache = gene.config['generation']['default_cache']

    if options.tiles is not None and options.role not in [
            'local', 'master'
    ]:  # pragma: no cover
        exit("The --tiles option work only with role local or master")

    try:
        if options.role == 'slave':
            generate = Generate()
            generate.gene(options, gene)
        elif (options.layer):
            generate = Generate()
            generate.gene(options, gene, options.layer)
        elif options.get_bbox:  # pragma: no cover
            exit("With --get-bbox option we needs to specify a layer")
        elif options.get_hash:  # pragma: no cover
            exit("With --get-hash option we needs to specify a layer")
        elif options.tiles:  # pragma: no cover
            exit("With --tiles option we needs to specify a layer")
        else:
            for layer in gene.config['generation'].get('default_layers',
                                                       gene.layers.keys()):
                generate = Generate()
                generate.gene(options, gene, layer)
    finally:
        if gene.error_file is not None:
            gene.error_file.close()
Exemplo n.º 19
0
def main():
    parser = ArgumentParser(
        description='Used to generate the tiles from Amazon EC2, '
        'and get the SQS queue status',
        prog=sys.argv[0])
    add_comon_options(parser)
    parser.add_argument('--deploy-config',
                        default=None,
                        dest="deploy_config",
                        metavar="FILE",
                        help='path to the deploy configuration file')
    parser.add_argument('--status',
                        default=False,
                        action="store_true",
                        help='display the SQS queue status and exit')
    parser.add_argument('--disable-geodata',
                        default=True,
                        action="store_false",
                        dest="geodata",
                        help='disable geodata synchronisation')
    parser.add_argument('--disable-code',
                        default=True,
                        action="store_false",
                        dest="deploy_code",
                        help='disable deploy application code')
    parser.add_argument('--disable-database',
                        default=True,
                        action="store_false",
                        dest="deploy_database",
                        help='disable deploy database')
    parser.add_argument('--disable-fillqueue',
                        default=True,
                        action="store_false",
                        dest="fill_queue",
                        help='disable queue filling')
    parser.add_argument('--disable-tilesgen',
                        default=True,
                        action="store_false",
                        dest="tiles_gen",
                        help='disable tile generation')
    parser.add_argument('--host',
                        default=None,
                        help='The host used to generate tiles')
    parser.add_argument('--shutdown',
                        default=False,
                        action="store_true",
                        help='Shut done the remote host after the task.')
    parser.add_argument('--wait',
                        default=False,
                        action="store_true",
                        help='Wait that all the tasks will finish.')
    parser.add_argument('--local',
                        default=False,
                        action="store_true",
                        help='Run the generation locally')

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.status:  # pragma: no cover
        status(options, gene)
        sys.exit(0)

    if 'ec2' not in gene.config:  # pragma: no cover
        print("EC2 not configured")
        sys.exit(1)

    if options.deploy_config is None:
        options.deploy_config = gene.config['ec2']['deploy_config']
    if options.geodata:
        options.geodata = not gene.config['ec2']['disable_geodata']
    if options.deploy_code:
        options.deploy_code = not gene.config['ec2']['disable_code']
    if options.deploy_database:
        options.deploy_database = not gene.config['ec2']['disable_database']
    if options.fill_queue:  # pragma: no cover
        options.fill_queue = not gene.config['ec2']['disable_fillqueue']
    if options.tiles_gen:  # pragma: no cover
        options.tiles_gen = not gene.config['ec2']['disable_tilesgen']

    # start aws
    if not options.host:
        # TODO not implemented yet
        host = aws_start(gene.config['ec2']['host_type'])  # pragma: no cover
    else:
        host = options.host

    if not options.local and options.geodata and 'geodata_folder' in gene.config[
            'ec2']:  # pragma: no cover
        print("==== Sync geodata ====")
        ssh_options = ''
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            ssh_options = gene.config['ec2']['ssh_options']
        # sync geodata
        run_local([
            'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
            gene.config['ec2']['geodata_folder'],
            host + ':' + gene.config['ec2']['geodata_folder']
        ])

    if options.deploy_code and not options.local:
        print("==== Sync and build code ====")

        cmd = [
            'rsync',
            '--delete',
        ]
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
            ssh_options = gene.config['ec2']['ssh_options']

        project_dir = gene.config['ec2']['code_folder']
        cmd += ['-r', '.', host + ':' + project_dir]
        run_local(cmd)

        for cmd in gene.config['ec2']['build_cmds']:
            run(options, cmd % environ, host, project_dir, gene)
        if 'apache_content' in gene.config[
                'ec2'] and 'apache_config' in gene.config['ec2']:
            run(
                options,
                'echo %s > %s' % (gene.config['ec2']['apache_content'],
                                  gene.config['ec2']['apache_config']), host,
                project_dir, gene)
        run(options, 'sudo apache2ctl graceful', host, project_dir, gene)

    # deploy
    if options.deploy_database and not options.local:
        _deploy(gene, host)

    if options.deploy_code or options.deploy_database \
            or options.geodata and not options.local:
        # TODO not implemented yet
        create_snapshot(host, gene)

    if options.time:
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'local'])
        arguments.extend(['--time', str(options.time)])

        project_dir = None if options.local else gene.config['ec2'][
            'code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    "%sgenerate_tiles %s" %
                    (_get_path(), ' '.join([str(a) for a in arguments])), host,
                    project_dir, gene))

        tiles_size = []
        times = []
        for p in processes:
            results = p.communicate()
            if results[1] != '':  # pragma: no cover
                logger.debug('ERROR: %s' % results[1])
            if PY3:
                results = [r.decode('utf-8') for r in results]
            results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
            results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
            for r in results[0].split('\n'):
                if r.startswith('time: '):
                    times.append(int(r.replace('time: ', '')))
                elif r.startswith('size: '):
                    tiles_size.append(int(r.replace('size: ', '')))

        if len(times) == 0:  # pragma: no cover
            logger.error("Not enough data")
            sys.exit(1)
        mean_time = reduce(lambda x, y: x + y,
                           [timedelta(microseconds=int(r))
                            for r in times], timedelta()) / len(times)**2
        mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0

        mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
                           0) / len(tiles_size)
        mean_size_kb = mean_size / 1024.0

        print('==== Time results ====')
        print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
        print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
        print('''config:
    cost:
        tileonly_generation_time: %0.3f
        tile_generation_time: %0.3f
        metatile_generation_time: 0
        tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))

        if options.shutdown:  # pragma: no cover
            run(options, 'sudo shutdown 0', host, project_dir, gene)
        sys.exit(0)

    if options.fill_queue and not options.local:  # pragma: no cover
        print("==== Till queue ====")
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'master', '--quiet'])

        project_dir = gene.config['ec2']['code_folder']
        run_remote_process(
            options, "%sgenerate_tiles %s" %
            (_get_path(), ' '.join([str(a) for a in arguments])), host,
            project_dir, gene)
        sleep(5)
        attributes = gene.get_sqs_queue().get_attributes()
        print("\rTiles to generate: %s/%s" % (
            attributes['ApproximateNumberOfMessages'],
            attributes['ApproximateNumberOfMessagesNotVisible'],
        ))

    if options.tiles_gen:  # pragma: no cover
        print("==== Generate tiles ====")

        if options.wait and not options.local:
            print("")

            class Status(Thread):
                def run(self):  # pragma: no cover
                    while True:
                        attributes = gene.get_sqs_queue().get_attributes()
                        print("\rTiles to generate/generating: %s/%s" % (
                            attributes['ApproximateNumberOfMessages'],
                            attributes['ApproximateNumberOfMessagesNotVisible'],
                        ))

                        sleep(1)

            status_thread = Status()
            status_thread.setDaemon(True)
            status_thread.start()

        arguments = _get_arguments(options)
        arguments.extend(['--quiet'])
        if not options.local:
            arguments.extend(['--role', 'slave'])

        project_dir = None if options.local else gene.config['ec2'][
            'code_folder']
        threads = []
        for i in range(gene.config['ec2']['number_process']):
            if options.local:
                threads.append(
                    run_local_process(
                        "%sgenerate_tiles --local-process-number %i %s" %
                        (_get_path(), i, ' '.join([str(a)
                                                   for a in arguments]))))
            else:
                run_remote_process(
                    "%sgenerate_tiles %s" %
                    (_get_path(), ' '.join([str(a) for a in arguments])), host,
                    project_dir, gene)

        print('Tile generation started')

        if options.shutdown:
            run(options, 'sudo shutdown 0')

        if options.wait and options.local:
            while len(threads) > 0:
                threads = [t for t in threads if t.is_alive()]
                sleep(1)

        if 'sns' in gene.config:
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(
                    gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
                    'host': socket.getfqdn(),
                    'cmd': ' '.join([quote(arg) for arg in sys.argv])
                }, "Tile generation controller")
Exemplo n.º 20
0
def main():
    stats.init_backends({})
    parser = ArgumentParser(
        description=
        'Used to generate the contextual file like the capabilities, the legends, '
        'the Apache and MapCache configuration',
        prog=sys.argv[0])
    add_comon_options(parser, tile_pyramid=False, no_geom=False)
    parser.add_argument('--status',
                        default=False,
                        action="store_true",
                        help='Display the SQS queue status and exit')
    parser.add_argument('--capabilities',
                        '--generate-wmts-capabilities',
                        default=False,
                        action='store_true',
                        help='Generate the WMTS Capabilities')
    parser.add_argument('--legends',
                        '--generate-legend-images',
                        default=False,
                        action='store_true',
                        dest='legends',
                        help='Generate the legend images')
    parser.add_argument('--openlayers',
                        '--generate-openlayers-testpage',
                        default=False,
                        action='store_true',
                        dest='openlayers',
                        help='Generate openlayers test page')
    parser.add_argument('--mapcache',
                        '--generate-mapcache-config',
                        default=False,
                        action='store_true',
                        dest='mapcache',
                        help='Generate MapCache configuration file')
    parser.add_argument('--apache',
                        '--generate-apache-config',
                        default=False,
                        action='store_true',
                        dest='apache',
                        help='Generate Apache configuration file')
    parser.add_argument(
        '--dump-config',
        default=False,
        action='store_true',
        help='Dump the used config with default values and exit')

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.status:  # pragma: no cover
        status(gene)
        sys.exit(0)

    if options.cache is None:
        options.cache = gene.config['generation']['default_cache']

    if options.dump_config:
        for layer in gene.config['layers'].values():
            gene.init_layer(layer, options)
        _validate_generate_wmts_capabilities(gene.caches[options.cache], True)
        for grid in gene.config['grids'].values():
            if 'obj' in grid:
                del grid['obj']
        print(yaml.dump(gene.config))
        sys.exit(0)

    if options.legends:
        _generate_legend_images(gene)

    if options.capabilities:
        _generate_wmts_capabilities(gene)

    if options.mapcache:
        _generate_mapcache_config(gene)

    if options.apache:
        _generate_apache_config(gene)

    if options.openlayers:
        _generate_openlayers(gene)
Exemplo n.º 21
0
def main():
    parser = ArgumentParser(
        description="Used to calculate the generation cost", prog=sys.argv[0])
    add_comon_options(parser, tile_pyramid=False, dimensions=True)
    parser.add_argument(
        "--cost-algo",
        "--calculate-cost-algorithm",
        default="area",
        dest="cost_algo",
        choices=("area", "count"),
        help=
        "The algorithm use to calculate the cost default base on the 'area' "
        "of the generation geometry, can also be 'count', to be base on number of tiles to generate.",
    )

    options = parser.parse_args()
    gene = TileGeneration(options.config,
                          options,
                          layer_name=options.layer,
                          base_config={"cost": {}},
                          multi_thread=False)

    all_size = 0
    tile_size = 0
    all_tiles = 0
    if options.layer:
        layer = gene.layers[options.layer]
        (all_size, all_time, all_price,
         all_tiles) = _calculate_cost(gene, layer, options)
        tile_size = layer["cost"]["tile_size"] / (1024.0 * 1024)
    else:
        all_time = timedelta()
        all_price = 0
        for layer_name in gene.config["generation"]["default_layers"]:
            print("")
            print("===== {} =====".format(layer_name))
            layer = gene.layers[layer_name]
            gene.init_layer(layer, options)
            (size, time, price, tiles) = _calculate_cost(gene, layer, options)
            tile_size += layer["cost"]["tile_size"] / (1024.0 * 1024)
            all_time += time
            all_price += price
            all_size += size
            all_tiles += tiles

        print("")
        print("===== GLOBAL =====")
        print("Total number of tiles: {}".format(all_tiles))
        print("Total generation time: {} [d h:mm:ss]".format(
            (duration_format(all_time))))
        print("Total generation cost: {0:0.2f} [$]".format(all_price))
    print("")
    print("S3 Storage: {0:0.2f} [$/month]".format(
        all_size * gene.config["cost"]["s3"]["storage"] /
        (1024.0 * 1024 * 1024)))
    print("S3 get: {0:0.2f} [$/month]".format(
        (gene.config["cost"]["s3"]["get"] *
         gene.config["cost"]["request_per_layers"] / 10000.0 +
         gene.config["cost"]["s3"]["download"] *
         gene.config["cost"]["request_per_layers"] * tile_size)))
    #    if 'cloudfront' in gene.config['cost']:
    #        print('CloudFront: %0.2f [$/month]' % ()
    #            gene.config['cost']['cloudfront']['get'] *
    #            gene.config['cost']['request_per_layers'] / 10000.0 +
    #            gene.config['cost']['cloudfront']['download'] *
    #            gene.config['cost']['request_per_layers'] * tile_size)
    sys.exit(0)
Exemplo n.º 22
0
def main():
    stats.init_backends({})
    parser = ArgumentParser(description="Used to generate the tiles",
                            prog=sys.argv[0])
    add_comon_options(parser, dimensions=True)
    parser.add_argument(
        "--get-hash",
        metavar="TILE",
        help="get the empty tiles hash, use the specified TILE z/x/y")
    parser.add_argument(
        "--get-bbox",
        metavar="TILE",
        help=
        "get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles",
    )
    parser.add_argument(
        "--role",
        default="local",
        choices=("local", "master", "slave"),
        help=
        "local/master/slave, master to file the queue and slave to generate the tiles",
    )
    parser.add_argument("--local-process-number",
                        default=None,
                        help="The number of process that we run in parallel")
    parser.add_argument("--detach",
                        default=False,
                        action="store_true",
                        help="run detached from the terminal")
    parser.add_argument("--daemon",
                        default=False,
                        action="store_true",
                        help="run continuously as a daemon")
    parser.add_argument(
        "--tiles",
        metavar="FILE",
        help=
        "Generate the tiles from a tiles file, use the format z/x/y, or z/x/y:+n/+n for metatiles",
    )

    options = parser.parse_args()

    if options.detach:
        detach()  # pragma: no cover

    gene = TileGeneration(options.config,
                          options,
                          multi_thread=options.get_hash is None)

    if (options.get_hash is None and options.get_bbox is None
            and "authorised_user" in gene.config["generation"]
            and gene.config["generation"]["authorised_user"] != getuser()):
        sys.exit("not authorised, authorised user is: {}.".format(
            gene.config["generation"]["authorised_user"]))

    if options.cache is None:
        options.cache = gene.config["generation"]["default_cache"]

    if options.tiles is not None and options.role not in [
            "local", "master"
    ]:  # pragma: no cover
        sys.exit("The --tiles option work only with role local or master")

    try:
        generate = Generate(options, gene)
        if options.role == "slave":
            generate.gene()
        elif options.layer:
            generate.gene(gene.layers[options.layer])
        elif options.get_bbox:  # pragma: no cover
            sys.exit("With --get-bbox option you need to specify a layer")
        elif options.get_hash:  # pragma: no cover
            sys.exit("With --get-hash option you need to specify a layer")
        elif options.tiles:  # pragma: no cover
            sys.exit("With --tiles option you need to specify a layer")
        else:
            for layer in gene.config["generation"].get("default_layers",
                                                       gene.layers.keys()):
                generate.gene(gene.layers[layer])
    finally:
        gene.close()
Exemplo n.º 23
0
def main():
    parser = ArgumentParser(
        description="Used to generate the contextual file like the capabilities, the legends, "
        "the Apache and MapCache configuration",
        prog=sys.argv[0],
    )
    add_comon_options(parser, tile_pyramid=False, no_geom=False)
    parser.add_argument(
        "--capabilities",
        "--generate-wmts-capabilities",
        default=False,
        action="store_true",
        help="Generate the WMTS Capabilities",
    )
    parser.add_argument(
        "--legends",
        "--generate-legend-images",
        default=False,
        action="store_true",
        dest="legends",
        help="Generate the legend images",
    )
    parser.add_argument(
        "--openlayers",
        "--generate-openlayers-test-page",
        default=False,
        action="store_true",
        dest="openlayers",
        help="Generate openlayers test page",
    )
    parser.add_argument(
        "--mapcache",
        "--generate-mapcache-config",
        default=False,
        action="store_true",
        dest="mapcache",
        help="Generate MapCache configuration file",
    )
    parser.add_argument(
        "--apache",
        "--generate-apache-config",
        default=False,
        action="store_true",
        dest="apache",
        help="Generate Apache configuration file",
    )
    parser.add_argument(
        "--dump-config", default=False, action="store_true", help="Dump the used config with default values and exit"
    )

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.cache is None:
        options.cache = gene.config["generation"]["default_cache"]

    if options.dump_config:
        for layer in gene.config["layers"].keys():
            gene.set_layer(layer, options)
            validate_calculate_cost(gene)
        _validate_generate_wmts_capabilities(gene, gene.caches[options.cache])
        gene.validate_mapcache_config()
        gene.validate_apache_config()
        _validate_generate_openlayers(gene)
        for grid in gene.config["grids"].values():
            if "obj" in grid:
                del grid["obj"]
        print(yaml.dump(gene.config))
        sys.exit(0)

    if options.legends:
        _generate_legend_images(gene)

    if options.capabilities:
        _generate_wmts_capabilities(gene)

    if options.mapcache:
        _generate_mapcache_config(gene)

    if options.apache:
        _generate_apache_config(gene)

    if options.openlayers:
        _generate_openlayers(gene)