コード例 #1
0
    def test_validate_type(self):
        class Opt:
            quiet = False
            verbose = False
            debug = False
            test = 0
            zoom = None

        gene = TileGeneration("tilegeneration/test.yaml", Opt())
        obj = {"value": 1}
        self.assertEqual(gene.validate(obj, "object", "value", int), False)
        self.assertEqual(obj["value"], 1)

        obj = {"value": 1.0}
        self.assertEqual(gene.validate(obj, "object", "value", int), True)

        obj = {"value": "1 + 1"}
        self.assertEqual(gene.validate(obj, "object", "value", int), False)
        self.assertEqual(obj["value"], 2)

        obj = {"value": "1 * 1.5"}
        self.assertEqual(gene.validate(obj, "object", "value", int), False)
        self.assertEqual(obj["value"], 2)

        obj = {"value": "a"}
        self.assertEqual(gene.validate(obj, "object", "value", int), True)

        obj = {"value": {}}
        self.assertEqual(gene.validate(obj, "object", "value", int), True)

        obj = {"value": []}
        self.assertEqual(gene.validate(obj, "object", "value", int), True)

        obj = {"value": 1}
        self.assertEqual(gene.validate(obj, "object", "value", float), False)
        self.assertEqual(obj["value"], 1.0)

        obj = {"value": 1.0}
        self.assertEqual(gene.validate(obj, "object", "value", float), False)
        self.assertEqual(obj["value"], 1.0)

        obj = {"value": "1 + 1"}
        self.assertEqual(gene.validate(obj, "object", "value", float), False)
        self.assertEqual(obj["value"], 2.0)

        obj = {"value": "1 * 1.5"}
        self.assertEqual(gene.validate(obj, "object", "value", float), False)
        self.assertEqual(obj["value"], 1.5)

        obj = {"value": "a"}
        self.assertEqual(gene.validate(obj, "object", "value", float), True)

        obj = {"value": {}}
        self.assertEqual(gene.validate(obj, "object", "value", float), True)

        obj = {"value": []}
        self.assertEqual(gene.validate(obj, "object", "value", float), True)
コード例 #2
0
ファイル: amazon.py プロジェクト: gberaudo/tilecloud-chain
def main():
    parser = ArgumentParser(
        description='Used to generate the tiles from Amazon EC2, '
        'and get the SQS queue status',
        prog='./buildout/bin/generate_amazon'
    )
    add_comon_options(parser)
    parser.add_argument(
        '--deploy-config', default=None, dest="deploy_config", metavar="FILE",
        help='path to the deploy configuration file'
    )
    parser.add_argument(
        '--status', default=False, action="store_true",
        help='display the SQS queue status and exit'
    )
    parser.add_argument(
        '--disable-geodata', default=True, action="store_false", dest="geodata",
        help='disable geodata synchronisation'
    )
    parser.add_argument(
        '--disable-code', default=True, action="store_false", dest="deploy_code",
        help='disable deploy application code'
    )
    parser.add_argument(
        '--disable-database', default=True, action="store_false", dest="deploy_database",
        help='disable deploy database'
    )
    parser.add_argument(
        '--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
        help='disable queue filling'
    )
    parser.add_argument(
        '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
        help='disable tile generation'
    )
    parser.add_argument(
        '--host', default=None,
        help='The host used to generate tiles'
    )
    parser.add_argument(
        '--shutdown', default=False, action="store_true",
        help='Shut done the remote host after the task.'
    )

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.status:  # pragma: no cover
        status(options, gene)
        sys.exit(0)

    if 'ec2' not in gene.config:  # pragma: no cover
        print "EC2 not configured"
        sys.exit(1)

    if options.deploy_config is None:
        options.deploy_config = gene.config['ec2']['deploy_config']
    if options.geodata:
        options.geodata = not gene.config['ec2']['disable_geodata']
    if options.deploy_code:
        options.deploy_code = not gene.config['ec2']['disable_code']
    if options.deploy_database:
        options.deploy_database = not gene.config['ec2']['disable_database']
    if options.fill_queue:  # pragma: no cover
        options.fill_queue = not gene.config['ec2']['disable_fillqueue']
    if options.tiles_gen:  # pragma: no cover
        options.tiles_gen = not gene.config['ec2']['disable_tilesgen']

    # start aws
    if not options.host:
        # TODO not implemented yet
        host = aws_start(gene.config['ec2']['host_type'])  # pragma: no cover
    else:
        host = options.host

    if options.geodata and 'geodata_folder' in gene.config['ec2']:
        print "==== Sync geodata ===="
        ssh_options = ''
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            ssh_options = gene.config['ec2']['ssh_options']
        # sync geodata
        run_local([
            'rsync', '--delete', '-e', 'ssh ' + ssh_options,
            '-r', gene.config['ec2']['geodata_folder'],
            host + ':' + gene.config['ec2']['geodata_folder']
        ])

    if options.deploy_code:
        print "==== Sync and build code ===="
        error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
        if error:
            exit(1)  # pragma: no cover

        cmd = ['rsync', '--delete', ]
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
            ssh_options = gene.config['ec2']['ssh_options']

        project_dir = gene.config['ec2']['code_folder']
        cmd += ['-r', '.', host + ':' + project_dir]
        run_local(cmd)

        for cmd in gene.config['ec2']['build_cmds']:
            run_remote(cmd, host, project_dir, gene)
        if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
            run_remote(
                'echo %s > %s' % (
                    gene.config['ec2']['apache_content'],
                    gene.config['ec2']['apache_config']
                ), host, project_dir, gene
            )
        run_remote('sudo apache2ctl graceful', host, project_dir, gene)

    # deploy
    if options.deploy_database:
        _deploy(gene, host)

    if options.deploy_code or options.deploy_database \
            or options.geodata:
        # TODO not implemented yet
        create_snapshot(host, gene)

    if options.time:
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'local'])
        arguments.extend(['--time', str(options.time)])

        project_dir = gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    './buildout/bin/generate_tiles ' +
                    ' '.join([str(a) for a in arguments]), host, project_dir, gene
                )
            )

        tiles_size = []
        times = []
        for p in processes:
            results = p.communicate()
            if results[1] != '':  # pragma: no cover
                logger.debug('ERROR: %s' % results[1])
            results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
            results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
            for r in results[0].split('\n'):
                if r.startswith('time: '):
                    times.append(int(r.replace('time: ', '')))
                elif r.startswith('size: '):
                    tiles_size.append(int(r.replace('size: ', '')))

        if len(times) == 0:  # pragma: no cover
            logger.error("Not enough data")
            sys.exit(1)
        mean_time = reduce(
            lambda x, y: x + y,
            [timedelta(microseconds=int(r)) for r in times],
            timedelta()
        ) / len(times) ** 2
        mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0

        mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
        mean_size_kb = mean_size / 1024.0

        print '==== Time results ===='
        print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
        print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
        print '''config:
    cost:
        tileonly_generation_time: %0.3f
        tile_generation_time: %0.3f
        metatile_generation_time: 0
        tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)

        if options.shutdown:  # pragma: no cover
            run_remote('sudo shutdown 0', host, project_dir, gene)
        sys.exit(0)

    if options.fill_queue:  # pragma: no cover
        print "==== Till queue ===="
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'master'])

        project_dir = gene.config['ec2']['code_folder']
        run_remote(
            './buildout/bin/generate_tiles ' +
            ' '.join([str(a) for a in arguments]), host, project_dir, gene
        )

    if options.tiles_gen:  # pragma: no cover
        print "==== Generate tiles ===="
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'slave'])
        arguments.append("--daemonize")

        project_dir = gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    './buildout/bin/generate_tiles ' +
                    ' '.join([str(a) for a in arguments]), host, project_dir, gene)
            )

        if options.shutdown:
            for p in processes:
                p.communicate()  # wait process end
        else:
            print 'Tile generation started in background'

        if options.shutdown:
            run_remote('sudo shutdown 0')

        if 'sns' in gene.config:
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'],
                """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
                {
                    'host': socket.getfqdn(),
                    'cmd': ' '.join([quote(arg) for arg in sys.argv])
                },
                "Tile generation controller")
コード例 #3
0
ファイル: amazon.py プロジェクト: danduk82/tilecloud-chain
def main():
    parser = ArgumentParser(
        description='Used to generate the tiles from Amazon EC2, '
        'and get the SQS queue status',
        prog=sys.argv[0]
    )
    add_comon_options(parser)
    parser.add_argument(
        '--deploy-config', default=None, dest="deploy_config", metavar="FILE",
        help='path to the deploy configuration file'
    )
    parser.add_argument(
        '--status', default=False, action="store_true",
        help='display the SQS queue status and exit'
    )
    parser.add_argument(
        '--disable-geodata', default=True, action="store_false", dest="geodata",
        help='disable geodata synchronisation'
    )
    parser.add_argument(
        '--disable-code', default=True, action="store_false", dest="deploy_code",
        help='disable deploy application code'
    )
    parser.add_argument(
        '--disable-database', default=True, action="store_false", dest="deploy_database",
        help='disable deploy database'
    )
    parser.add_argument(
        '--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
        help='disable queue filling'
    )
    parser.add_argument(
        '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
        help='disable tile generation'
    )
    parser.add_argument(
        '--host', default=None,
        help='The host used to generate tiles'
    )
    parser.add_argument(
        '--shutdown', default=False, action="store_true",
        help='Shut done the remote host after the task.'
    )
    parser.add_argument(
        '--wait', default=False, action="store_true",
        help='Wait that all the tasks will finish.'
    )
    parser.add_argument(
        '--local', default=False, action="store_true",
        help='Run the generation locally'
    )

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.status:  # pragma: no cover
        status(options, gene)
        sys.exit(0)

    if 'ec2' not in gene.config:  # pragma: no cover
        print("EC2 not configured")
        sys.exit(1)

    if options.deploy_config is None:
        options.deploy_config = gene.config['ec2']['deploy_config']
    if options.geodata:
        options.geodata = not gene.config['ec2']['disable_geodata']
    if options.deploy_code:
        options.deploy_code = not gene.config['ec2']['disable_code']
    if options.deploy_database:
        options.deploy_database = not gene.config['ec2']['disable_database']
    if options.fill_queue:  # pragma: no cover
        options.fill_queue = not gene.config['ec2']['disable_fillqueue']
    if options.tiles_gen:  # pragma: no cover
        options.tiles_gen = not gene.config['ec2']['disable_tilesgen']

    # start aws
    if not options.host:
        # TODO not implemented yet
        host = aws_start(gene.config['ec2']['host_type'])  # pragma: no cover
    else:
        host = options.host

    if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']:
        print("==== Sync geodata ====")
        ssh_options = ''
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            ssh_options = gene.config['ec2']['ssh_options']
        # sync geodata
        run_local([
            'rsync', '--delete', '-e', 'ssh ' + ssh_options,
            '-r', gene.config['ec2']['geodata_folder'],
            host + ':' + gene.config['ec2']['geodata_folder']
        ])

    if options.deploy_code and not options.local:
        print("==== Sync and build code ====")
        error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
        if error:
            exit(1)  # pragma: no cover

        cmd = ['rsync', '--delete', ]
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
            ssh_options = gene.config['ec2']['ssh_options']

        project_dir = gene.config['ec2']['code_folder']
        cmd += ['-r', '.', host + ':' + project_dir]
        run_local(cmd)

        for cmd in gene.config['ec2']['build_cmds']:
            run(options, cmd % environ, host, project_dir, gene)
        if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
            run(
                options,
                'echo %s > %s' % (
                    gene.config['ec2']['apache_content'],
                    gene.config['ec2']['apache_config']
                ), host, project_dir, gene
            )
        run(options, 'sudo apache2ctl graceful', host, project_dir, gene)

    # deploy
    if options.deploy_database and not options.local:
        _deploy(gene, host)

    if options.deploy_code or options.deploy_database \
            or options.geodata and not options.local:
        # TODO not implemented yet
        create_snapshot(host, gene)

    if options.time:
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'local'])
        arguments.extend(['--time', str(options.time)])

        project_dir = None if options.local else gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    "%sgenerate_tiles %s" % (
                        _get_path(),
                        ' '.join([str(a) for a in arguments])
                    ), host, project_dir, gene
                )
            )

        tiles_size = []
        times = []
        for p in processes:
            results = p.communicate()
            if results[1] != '':  # pragma: no cover
                logger.debug('ERROR: %s' % results[1])
            if PY3:
                results = [r.decode('utf-8') for r in results]
            results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
            results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
            for r in results[0].split('\n'):
                if r.startswith('time: '):
                    times.append(int(r.replace('time: ', '')))
                elif r.startswith('size: '):
                    tiles_size.append(int(r.replace('size: ', '')))

        if len(times) == 0:  # pragma: no cover
            logger.error("Not enough data")
            sys.exit(1)
        mean_time = reduce(
            lambda x, y: x + y,
            [timedelta(microseconds=int(r)) for r in times],
            timedelta()
        ) / len(times) ** 2
        mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0

        mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
        mean_size_kb = mean_size / 1024.0

        print('==== Time results ====')
        print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
        print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
        print('''config:
    cost:
        tileonly_generation_time: %0.3f
        tile_generation_time: %0.3f
        metatile_generation_time: 0
        tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))

        if options.shutdown:  # pragma: no cover
            run(options, 'sudo shutdown 0', host, project_dir, gene)
        sys.exit(0)

    if options.fill_queue and not options.local:  # pragma: no cover
        print("==== Till queue ====")
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'master', '--quiet'])

        project_dir = gene.config['ec2']['code_folder']
        run_remote_process(
            options,
            "%sgenerate_tiles %s" % (
                _get_path(),
                ' '.join([str(a) for a in arguments])
            ), host, project_dir, gene
        )
        sleep(5)
        attributes = gene.get_sqs_queue().get_attributes()
        print(
            "\rTiles to generate: %s/%s" % (
                attributes['ApproximateNumberOfMessages'],
                attributes['ApproximateNumberOfMessagesNotVisible'],
            )
        )

    if options.tiles_gen:  # pragma: no cover
        print("==== Generate tiles ====")

        if options.wait and not options.local:
            print("")

            class Status(Thread):
                def run(self):  # pragma: no cover
                    while True:
                        attributes = gene.get_sqs_queue().get_attributes()
                        print(
                            "\rTiles to generate/generating: %s/%s" % (
                                attributes['ApproximateNumberOfMessages'],
                                attributes['ApproximateNumberOfMessagesNotVisible'],
                            )
                        )

                        sleep(1)
            status_thread = Status()
            status_thread.setDaemon(True)
            status_thread.start()

        arguments = _get_arguments(options)
        arguments.extend(['--quiet'])
        if not options.local:
            arguments.extend(['--role', 'slave'])

        project_dir = None if options.local else gene.config['ec2']['code_folder']
        threads = []
        for i in range(gene.config['ec2']['number_process']):
            if options.local:
                threads.append(run_local_process(
                    "%sgenerate_tiles --local-process-number %i %s" % (
                        _get_path(),
                        i, ' '.join([str(a) for a in arguments])
                    )
                ))
            else:
                run_remote_process(
                    "%sgenerate_tiles %s" % (
                        _get_path(),
                        ' '.join([str(a) for a in arguments])
                    ), host, project_dir, gene
                )

        print('Tile generation started')

        if options.shutdown:
            run(options, 'sudo shutdown 0')

        if options.wait and options.local:
            while len(threads) > 0:
                threads = [t for t in threads if t.is_alive()]
                sleep(1)

        if 'sns' in gene.config:
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'],
                """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
                {
                    'host': socket.getfqdn(),
                    'cmd': ' '.join([quote(arg) for arg in sys.argv])
                },
                "Tile generation controller"
            )
コード例 #4
0
def main():
    parser = ArgumentParser(
        description='Used to generate the tiles from Amazon EC2, '
        'and get the SQS queue status',
        prog='./buildout/bin/generate_amazon')
    add_comon_options(parser)
    parser.add_argument('--deploy-config',
                        default=None,
                        dest="deploy_config",
                        metavar="FILE",
                        help='path to the deploy configuration file')
    parser.add_argument('--status',
                        default=False,
                        action="store_true",
                        help='display the SQS queue status and exit')
    parser.add_argument('--disable-geodata',
                        default=True,
                        action="store_false",
                        dest="geodata",
                        help='disable geodata synchronisation')
    parser.add_argument('--disable-code',
                        default=True,
                        action="store_false",
                        dest="deploy_code",
                        help='disable deploy application code')
    parser.add_argument('--disable-database',
                        default=True,
                        action="store_false",
                        dest="deploy_database",
                        help='disable deploy database')
    parser.add_argument('--disable-fillqueue',
                        default=True,
                        action="store_false",
                        dest="fill_queue",
                        help='disable queue filling')
    parser.add_argument('--disable-tilesgen',
                        default=True,
                        action="store_false",
                        dest="tiles_gen",
                        help='disable tile generation')
    parser.add_argument('--host',
                        default=None,
                        help='The host used to generate tiles')
    parser.add_argument('--shutdown',
                        default=False,
                        action="store_true",
                        help='Shut done the remote host after the task.')

    options = parser.parse_args()
    gene = TileGeneration(options.config, options, layer_name=options.layer)

    if options.status:  # pragma: no cover
        status(options, gene)
        sys.exit(0)

    if 'ec2' not in gene.config:  # pragma: no cover
        print "EC2 not configured"
        sys.exit(1)

    if options.deploy_config is None:
        options.deploy_config = gene.config['ec2']['deploy_config']
    if options.geodata:
        options.geodata = not gene.config['ec2']['disable_geodata']
    if options.deploy_code:
        options.deploy_code = not gene.config['ec2']['disable_code']
    if options.deploy_database:
        options.deploy_database = not gene.config['ec2']['disable_database']
    if options.fill_queue:  # pragma: no cover
        options.fill_queue = not gene.config['ec2']['disable_fillqueue']
    if options.tiles_gen:  # pragma: no cover
        options.tiles_gen = not gene.config['ec2']['disable_tilesgen']

    # start aws
    if not options.host:
        # TODO not implemented yet
        host = aws_start(gene.config['ec2']['host_type'])  # pragma: no cover
    else:
        host = options.host

    if options.geodata and 'geodata_folder' in gene.config['ec2']:
        print "==== Sync geodata ===="
        ssh_options = ''
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            ssh_options = gene.config['ec2']['ssh_options']
        # sync geodata
        run_local([
            'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
            gene.config['ec2']['geodata_folder'],
            host + ':' + gene.config['ec2']['geodata_folder']
        ])

    if options.deploy_code:
        print "==== Sync and build code ===="
        error = gene.validate(gene.config['ec2'],
                              'ec2',
                              'code_folder',
                              required=True)
        if error:
            exit(1)  # pragma: no cover

        cmd = [
            'rsync',
            '--delete',
        ]
        if 'ssh_options' in gene.config['ec2']:  # pragma: no cover
            cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
            ssh_options = gene.config['ec2']['ssh_options']

        project_dir = gene.config['ec2']['code_folder']
        cmd += ['-r', '.', host + ':' + project_dir]
        run_local(cmd)

        for cmd in gene.config['ec2']['build_cmds']:
            run_remote(cmd, host, project_dir, gene)
        if 'apache_content' in gene.config[
                'ec2'] and 'apache_config' in gene.config['ec2']:
            run_remote(
                'echo %s > %s' % (gene.config['ec2']['apache_content'],
                                  gene.config['ec2']['apache_config']), host,
                project_dir, gene)
        run_remote('sudo apache2ctl graceful', host, project_dir, gene)

    # deploy
    if options.deploy_database:
        _deploy(gene, host)

    if options.deploy_code or options.deploy_database \
            or options.geodata:
        # TODO not implemented yet
        create_snapshot(host, gene)

    if options.time:
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'local'])
        arguments.extend(['--time', str(options.time)])

        project_dir = gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    './buildout/bin/generate_tiles ' +
                    ' '.join([str(a) for a in arguments]), host, project_dir,
                    gene))

        tiles_size = []
        times = []
        for p in processes:
            results = p.communicate()
            if results[1] != '':  # pragma: no cover
                logger.debug('ERROR: %s' % results[1])
            results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
            results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
            for r in results[0].split('\n'):
                if r.startswith('time: '):
                    times.append(int(r.replace('time: ', '')))
                elif r.startswith('size: '):
                    tiles_size.append(int(r.replace('size: ', '')))

        if len(times) == 0:  # pragma: no cover
            logger.error("Not enough data")
            sys.exit(1)
        mean_time = reduce(lambda x, y: x + y,
                           [timedelta(microseconds=int(r))
                            for r in times], timedelta()) / len(times)**2
        mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0

        mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
                           0) / len(tiles_size)
        mean_size_kb = mean_size / 1024.0

        print '==== Time results ===='
        print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
        print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
        print '''config:
    cost:
        tileonly_generation_time: %0.3f
        tile_generation_time: %0.3f
        metatile_generation_time: 0
        tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)

        if options.shutdown:  # pragma: no cover
            run_remote('sudo shutdown 0', host, project_dir, gene)
        sys.exit(0)

    if options.fill_queue:  # pragma: no cover
        print "==== Till queue ===="
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'master'])

        project_dir = gene.config['ec2']['code_folder']
        run_remote(
            './buildout/bin/generate_tiles ' +
            ' '.join([str(a) for a in arguments]), host, project_dir, gene)

    if options.tiles_gen:  # pragma: no cover
        print "==== Generate tiles ===="
        # TODO test
        arguments = _get_arguments(options)
        arguments.extend(['--role', 'slave'])
        arguments.append("--daemonize")

        project_dir = gene.config['ec2']['code_folder']
        processes = []
        for i in range(gene.config['ec2']['number_process']):
            processes.append(
                run_remote_process(
                    './buildout/bin/generate_tiles ' +
                    ' '.join([str(a) for a in arguments]), host, project_dir,
                    gene))

        if options.shutdown:
            for p in processes:
                p.communicate()  # wait process end
        else:
            print 'Tile generation started in background'

        if options.shutdown:
            run_remote('sudo shutdown 0')

        if 'sns' in gene.config:
            if 'region' in gene.config['sns']:
                connection = sns.connect_to_region(
                    gene.config['sns']['region'])
            else:
                connection = boto.connect_sns()
            connection.publish(
                gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
                    'host': socket.getfqdn(),
                    'cmd': ' '.join([quote(arg) for arg in sys.argv])
                }, "Tile generation controller")
コード例 #5
0
    def test_validate_type(self):
        class Opt:
            quiet = False
            verbose = False
            debug = False
            test = 0
            zoom = None
        gene = TileGeneration('tilegeneration/test.yaml', Opt())
        obj = {'value': 1}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
        self.assertEquals(obj['value'],  1)

        obj = {'value': 1.0}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), True)

        obj = {'value': '1 + 1'}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
        self.assertEquals(obj['value'],  2)

        obj = {'value': '1 * 1.5'}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
        self.assertEquals(obj['value'],  2)

        obj = {'value': 'a'}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), True)

        obj = {'value': {}}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), True)

        obj = {'value': []}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), True)

        obj = {'value': 1}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
        self.assertEquals(obj['value'],  1.0)

        obj = {'value': 1.0}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
        self.assertEquals(obj['value'],  1.0)

        obj = {'value': '1 + 1'}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
        self.assertEquals(obj['value'],  2.0)

        obj = {'value': '1 * 1.5'}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
        self.assertEquals(obj['value'],  1.5)

        obj = {'value': 'a'}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), True)

        obj = {'value': {}}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), True)

        obj = {'value': []}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), True)
コード例 #6
0
    def test_validate_type(self):
        class Opt:
            quiet = False
            verbose = False
            debug = False
            test = 0
            zoom = None

        gene = TileGeneration('tilegeneration/test.yaml', Opt())
        obj = {'value': 1}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
        self.assertEquals(obj['value'], 1)

        obj = {'value': 1.0}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), True)

        obj = {'value': '1 + 1'}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
        self.assertEquals(obj['value'], 2)

        obj = {'value': '1 * 1.5'}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), False)
        self.assertEquals(obj['value'], 2)

        obj = {'value': 'a'}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), True)

        obj = {'value': {}}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), True)

        obj = {'value': []}
        self.assertEquals(gene.validate(obj, 'object', 'value', int), True)

        obj = {'value': 1}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
        self.assertEquals(obj['value'], 1.0)

        obj = {'value': 1.0}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
        self.assertEquals(obj['value'], 1.0)

        obj = {'value': '1 + 1'}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
        self.assertEquals(obj['value'], 2.0)

        obj = {'value': '1 * 1.5'}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), False)
        self.assertEquals(obj['value'], 1.5)

        obj = {'value': 'a'}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), True)

        obj = {'value': {}}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), True)

        obj = {'value': []}
        self.assertEquals(gene.validate(obj, 'object', 'value', float), True)