def run_local_process(cmd): if type(cmd) != list: cmd = cmd.split(' ') logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd])) task = Run(cmd) task.start() return task
def run_remote_process(remote_cmd, host, project_dir, gene): cmd = ['ssh'] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd.extend(gene.config['ec2']['ssh_options'].split(' ')) if host is None: # pragma: no cover exit('host option is required.') cmd.append(host) env = '' if os.getenv('AWS_ACCESS_KEY_ID') and os.getenv('AWS_SECRET_ACCESS_KEY'): # pragma: no cover env = 'export AWS_ACCESS_KEY_ID=%(access_key)s;export AWS_SECRET_ACCESS_KEY=%(secret_key)s;' % { 'access_key': os.getenv('AWS_ACCESS_KEY_ID'), 'secret_key': os.getenv('AWS_SECRET_ACCESS_KEY'), } cmd.append( 'cd %(project_dir)s;' '%(env)s' '%(cmd)s' % { 'cmd': remote_cmd, 'env': env, 'project_dir': project_dir } ) logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd])) return Popen(cmd, stdout=PIPE, stderr=PIPE)
def run_local(cmd): if type(cmd) != list: cmd = cmd.split(' ') logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd])) result = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate() logger.info(result[0]) logger.error(result[1]) return result
def run_local(cmd): if type(cmd) != list: cmd = cmd.split(' ') logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd])) result = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate() if PY3: result = [r.decode('utf-8') for r in result] if len(result[0]) != 0: # pragma: no cover logger.info(result[0]) if len(result[1]) != 0: logger.error(result[1]) return result
def run_remote_process(remote_cmd, host, project_dir, gene): cmd = ['ssh'] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd.extend(gene.config['ec2']['ssh_options'].split(' ')) if host is None: # pragma: no cover exit('host option is required.') cmd.append(host) env = '' if os.getenv('AWS_ACCESS_KEY_ID') and os.getenv( 'AWS_SECRET_ACCESS_KEY'): # pragma: no cover env = 'export AWS_ACCESS_KEY_ID=%(access_key)s;export AWS_SECRET_ACCESS_KEY=%(secret_key)s;' % { 'access_key': os.getenv('AWS_ACCESS_KEY_ID'), 'secret_key': os.getenv('AWS_SECRET_ACCESS_KEY'), } cmd.append('cd %(project_dir)s;' '%(env)s' '%(cmd)s' % { 'cmd': remote_cmd, 'env': env, 'project_dir': project_dir }) logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd])) return Popen(cmd, stdout=PIPE, stderr=PIPE)
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog=sys.argv[0]) add_comon_options(parser) parser.add_argument('--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file') parser.add_argument('--status', default=False, action="store_true", help='display the SQS queue status and exit') parser.add_argument('--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation') parser.add_argument('--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code') parser.add_argument('--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database') parser.add_argument('--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling') parser.add_argument('--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation') parser.add_argument('--host', default=None, help='The host used to generate tiles') parser.add_argument('--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.') parser.add_argument('--wait', default=False, action="store_true", help='Wait that all the tasks will finish.') parser.add_argument('--local', default=False, action="store_true", help='Run the generation locally') options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print("EC2 not configured") sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if not options.local and options.geodata and 'geodata_folder' in gene.config[ 'ec2']: # pragma: no cover print("==== Sync geodata ====") ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code and not options.local: print("==== Sync and build code ====") cmd = [ 'rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run(options, cmd % environ, host, project_dir, gene) if 'apache_content' in gene.config[ 'ec2'] and 'apache_config' in gene.config['ec2']: run( options, 'echo %s > %s' % (gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config']), host, project_dir, gene) run(options, 'sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database and not options.local: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata and not options.local: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = None if options.local else gene.config['ec2'][ 'code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( "%sgenerate_tiles %s" % (_get_path(), ' '.join([str(a) for a in arguments])), host, project_dir, gene)) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) if PY3: results = [r.decode('utf-8') for r in results] results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce(lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta()) / len(times)**2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print('==== Time results ====') print('A tile is generated in: %0.3f [ms]' % mean_time_ms) print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)) print('''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)) if options.shutdown: # pragma: no cover run(options, 'sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue and not options.local: # pragma: no cover print("==== Till queue ====") # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master', '--quiet']) project_dir = gene.config['ec2']['code_folder'] run_remote_process( options, "%sgenerate_tiles %s" % (_get_path(), ' '.join([str(a) for a in arguments])), host, project_dir, gene) sleep(5) attributes = gene.get_sqs_queue().get_attributes() print("\rTiles to generate: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], )) if options.tiles_gen: # pragma: no cover print("==== Generate tiles ====") if options.wait and not options.local: print("") class Status(Thread): def run(self): # pragma: no cover while True: attributes = gene.get_sqs_queue().get_attributes() print("\rTiles to generate/generating: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], )) sleep(1) status_thread = Status() status_thread.setDaemon(True) status_thread.start() arguments = _get_arguments(options) arguments.extend(['--quiet']) if not options.local: arguments.extend(['--role', 'slave']) project_dir = None if options.local else gene.config['ec2'][ 'code_folder'] threads = [] for i in range(gene.config['ec2']['number_process']): if options.local: threads.append( run_local_process( "%sgenerate_tiles --local-process-number %i %s" % (_get_path(), i, ' '.join([str(a) for a in arguments])))) else: run_remote_process( "%sgenerate_tiles %s" % (_get_path(), ' '.join([str(a) for a in arguments])), host, project_dir, gene) print('Tile generation started') if options.shutdown: run(options, 'sudo shutdown 0') if options.wait and options.local: while len(threads) > 0: threads = [t for t in threads if t.is_alive()] sleep(1) if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller")
def _gene(self, options, gene, layer, dimensions={}): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() count_tiles_stored = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print("Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord)) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache, dimensions) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join( gene.layer['wmts_style'] for l in gene.layer['layers'].split(',')) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(dimensions) # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=gene.layer['layers'], srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': # pragma: no cover from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': count_tiles = gene.counter() gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub( '', tile.data.decode('utf-8') if PY3 else tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() if gene.layer['type'] != 'mapnik' or gene.layer[ 'output_format'] != 'grid': count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: # pragma: no cover gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() else: # pragma: no cover count_tiles = gene.counter() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()])), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( count_tiles.nb)) else: if meta: message += [ "Nb generated metatiles: {}".format( count_metatiles.nb), "Nb metatiles dropped: {}".format( count_metatiles_dropped.nb), ] message += [ "Nb generated tiles: {}".format(count_tiles.nb), "Nb tiles dropped: {}".format(count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format( gene.duration)), ] if count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(count_tiles_stored.size))) if count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / count_tiles.nb * 1000).seconds)) if count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( count_tiles_stored.size / count_tiles_stored.nb)) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name']), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
def _gene(self, options, gene, dimensions=None): if dimensions is None: # pragma: no cover dimensions = {} self.dimensions = dimensions self.count_metatiles = None self.count_metatiles_dropped = Count() self.count_tiles = Count() self.count_tiles_dropped = Count() self.count_tiles_stored = None self.sqs_tilestore = None self.cache_tilestore = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print("Tile bounds: [{},{},{},{}]".format(*default_int( gene.layer['grid_ref']['obj'].extent(tilecoord)))) exit() except ValueError as e: # pragma: no cover print( "Tile '{}' is not in the format 'z/x/y' or z/x/y:+n/+n\n{}" .format(options.get_bbox, repr(e))) exit(1) if options.get_hash: options.role = 'hash' options.test = 1 if options.role in ('master', 'slave'): # Create SQS queue self.sqs_tilestore = SQSTileStore( gene.get_sqs_queue(), on_empty=await_message if options.daemon else maybe_stop) # pragma: no cover if options.role in ('local', 'slave'): self.cache_tilestore = gene.get_tilesstore(options.cache, dimensions) if options.tiles: gene.set_store(TilesFileStore(options.tiles, options.layer)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.role in ('local', 'master') and 'logging' in gene.config: gene.imap( DatabaseLoggerInit(gene.config['logging'], options is not None and options.daemon)) if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(self.sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if gene.layer.get('meta'): gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '{}' is not in the format 'z/x/y'\n{}".format( options.get_hash, repr(e))) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() self.count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(self.sqs_tilestore) self.count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): gene.get( MultiTileStore({ name: self._get_tilestore_for_layer(layer, gene) for name, layer in gene.layers.items() }), 'Get tile') def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: {}".format( (self._re_rm_xml_tag.sub('', tile.error))) else: # pragma: no cover tile.error = "{} is not an image format, error: {}".format( tile.content_type, tile.error) return tile gene.imap(wrong_content_type_to_error) gene.add_error_filters() if options.role == 'hash': if gene.layer.get('meta', False): gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: droppers = {} for lname, layer in gene.layers.items(): if 'empty_metatile_detection' in layer: empty_tile = layer['empty_metatile_detection'] droppers[lname] = HashDropper( empty_tile['size'], empty_tile['hash'], store=self.cache_tilestore, queue_store=self.sqs_tilestore, count=self.count_metatiles_dropped, ) if droppers: gene.imap(MultiAction(droppers)) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) gene.imap(self.count_tiles) gene.process(key='pre_hash_post_process') if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: droppers = {} for lname, layer in gene.layers.items(): if 'empty_tile_detection' in layer: empty_tile = layer['empty_tile_detection'] droppers[lname] = HashDropper( empty_tile['size'], empty_tile['hash'], store=self.cache_tilestore, queue_store=self.sqs_tilestore, count=self.count_tiles_dropped, ) if droppers: gene.imap(MultiAction(droppers)) gene.process() else: # pragma: no cover self.count_tiles = gene.counter() if options.role in ('local', 'slave'): self.count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: {}\n'.format(len(tile.data))) return tile gene.imap(log_size) gene.put(self.cache_tilestore, "Store the tile") if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('{}\n'.format(tile.tilecoord)) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover def delete_from_store(tile): if hasattr(tile, 'metatile'): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: self.sqs_tilestore.delete_one(tile.metatile) else: self.sqs_tilestore.delete_one(tile) return True gene.ifilter(delete_from_store) if options.role in ('local', 'slave') and 'logging' in gene.config: gene.imap( DatabaseLogger(gene.config['logging'], options is not None and options.daemon)) gene.add_error_filters() message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write('time: {}\n'.format( ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds))) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if gene.layer is not None: message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " ({})".format(", ".join( ["=".join(d) for d in dimensions.items()]))), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( self.count_tiles.nb)) else: if gene.layer.get('meta'): message += [ "Nb generated metatiles: {}".format( self.count_metatiles.nb), "Nb metatiles dropped: {}".format( self.count_metatiles_dropped.nb), ] else: message = ["The tile generation is finish"] if options.role != "master": message += [ "Nb generated tiles: {}".format(self.count_tiles.nb), "Nb tiles dropped: {}".format(self.count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format( self.count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format( gene.duration)), ] if self.count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(self.count_tiles_stored.size))) if self.count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / self.count_tiles.nb * 1000).seconds)) if self.count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( self.count_tiles_stored.size / self.count_tiles_stored.nb)) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if self.cache_tilestore is not None and hasattr( self.cache_tilestore, 'connection'): self.cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name'] if gene. layer is not None else "(All layers)"), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation ({layer!s} - {role!s})".format( **{ 'role': options.role, 'layer': gene.layer['name'] if gene. layer is not None else "All layers" }))
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog='./buildout/bin/generate_amazon' ) add_comon_options(parser) parser.add_argument( '--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file' ) parser.add_argument( '--status', default=False, action="store_true", help='display the SQS queue status and exit' ) parser.add_argument( '--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation' ) parser.add_argument( '--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code' ) parser.add_argument( '--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database' ) parser.add_argument( '--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling' ) parser.add_argument( '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation' ) parser.add_argument( '--host', default=None, help='The host used to generate tiles' ) parser.add_argument( '--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.' ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print "EC2 not configured" sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if options.geodata and 'geodata_folder' in gene.config['ec2']: print "==== Sync geodata ====" ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code: print "==== Sync and build code ====" error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True) if error: exit(1) # pragma: no cover cmd = ['rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run_remote(cmd, host, project_dir, gene) if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']: run_remote( 'echo %s > %s' % ( gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config'] ), host, project_dir, gene ) run_remote('sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene ) ) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce( lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta() ) / len(times) ** 2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print '==== Time results ====' print 'A tile is generated in: %0.3f [ms]' % mean_time_ms print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb) print '''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb) if options.shutdown: # pragma: no cover run_remote('sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue: # pragma: no cover print "==== Till queue ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master']) project_dir = gene.config['ec2']['code_folder'] run_remote( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene ) if options.tiles_gen: # pragma: no cover print "==== Generate tiles ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'slave']) arguments.append("--daemonize") project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene) ) if options.shutdown: for p in processes: p.communicate() # wait process end else: print 'Tile generation started in background' if options.shutdown: run_remote('sudo shutdown 0') if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller")
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog='./buildout/bin/generate_amazon') add_comon_options(parser) parser.add_argument('--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file') parser.add_argument('--status', default=False, action="store_true", help='display the SQS queue status and exit') parser.add_argument('--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation') parser.add_argument('--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code') parser.add_argument('--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database') parser.add_argument('--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling') parser.add_argument('--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation') parser.add_argument('--host', default=None, help='The host used to generate tiles') parser.add_argument('--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.') options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print "EC2 not configured" sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if options.geodata and 'geodata_folder' in gene.config['ec2']: print "==== Sync geodata ====" ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code: print "==== Sync and build code ====" error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True) if error: exit(1) # pragma: no cover cmd = [ 'rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run_remote(cmd, host, project_dir, gene) if 'apache_content' in gene.config[ 'ec2'] and 'apache_config' in gene.config['ec2']: run_remote( 'echo %s > %s' % (gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config']), host, project_dir, gene) run_remote('sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene)) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce(lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta()) / len(times)**2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print '==== Time results ====' print 'A tile is generated in: %0.3f [ms]' % mean_time_ms print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb) print '''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb) if options.shutdown: # pragma: no cover run_remote('sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue: # pragma: no cover print "==== Till queue ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master']) project_dir = gene.config['ec2']['code_folder'] run_remote( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene) if options.tiles_gen: # pragma: no cover print "==== Generate tiles ====" # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'slave']) arguments.append("--daemonize") project_dir = gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( './buildout/bin/generate_tiles ' + ' '.join([str(a) for a in arguments]), host, project_dir, gene)) if options.shutdown: for p in processes: p.communicate() # wait process end else: print 'Tile generation started in background' if options.shutdown: run_remote('sudo shutdown 0') if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller")
def main(): parser = ArgumentParser( description='Used to generate the tiles from Amazon EC2, ' 'and get the SQS queue status', prog=sys.argv[0] ) add_comon_options(parser) parser.add_argument( '--deploy-config', default=None, dest="deploy_config", metavar="FILE", help='path to the deploy configuration file' ) parser.add_argument( '--status', default=False, action="store_true", help='display the SQS queue status and exit' ) parser.add_argument( '--disable-geodata', default=True, action="store_false", dest="geodata", help='disable geodata synchronisation' ) parser.add_argument( '--disable-code', default=True, action="store_false", dest="deploy_code", help='disable deploy application code' ) parser.add_argument( '--disable-database', default=True, action="store_false", dest="deploy_database", help='disable deploy database' ) parser.add_argument( '--disable-fillqueue', default=True, action="store_false", dest="fill_queue", help='disable queue filling' ) parser.add_argument( '--disable-tilesgen', default=True, action="store_false", dest="tiles_gen", help='disable tile generation' ) parser.add_argument( '--host', default=None, help='The host used to generate tiles' ) parser.add_argument( '--shutdown', default=False, action="store_true", help='Shut done the remote host after the task.' ) parser.add_argument( '--wait', default=False, action="store_true", help='Wait that all the tasks will finish.' ) parser.add_argument( '--local', default=False, action="store_true", help='Run the generation locally' ) options = parser.parse_args() gene = TileGeneration(options.config, options, layer_name=options.layer) if options.status: # pragma: no cover status(options, gene) sys.exit(0) if 'ec2' not in gene.config: # pragma: no cover print("EC2 not configured") sys.exit(1) if options.deploy_config is None: options.deploy_config = gene.config['ec2']['deploy_config'] if options.geodata: options.geodata = not gene.config['ec2']['disable_geodata'] if options.deploy_code: options.deploy_code = not gene.config['ec2']['disable_code'] if options.deploy_database: options.deploy_database = not gene.config['ec2']['disable_database'] if options.fill_queue: # pragma: no cover options.fill_queue = not gene.config['ec2']['disable_fillqueue'] if options.tiles_gen: # pragma: no cover options.tiles_gen = not gene.config['ec2']['disable_tilesgen'] # start aws if not options.host: # TODO not implemented yet host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover else: host = options.host if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']: print("==== Sync geodata ====") ssh_options = '' if 'ssh_options' in gene.config['ec2']: # pragma: no cover ssh_options = gene.config['ec2']['ssh_options'] # sync geodata run_local([ 'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r', gene.config['ec2']['geodata_folder'], host + ':' + gene.config['ec2']['geodata_folder'] ]) if options.deploy_code and not options.local: print("==== Sync and build code ====") error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True) if error: exit(1) # pragma: no cover cmd = ['rsync', '--delete', ] if 'ssh_options' in gene.config['ec2']: # pragma: no cover cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']] ssh_options = gene.config['ec2']['ssh_options'] project_dir = gene.config['ec2']['code_folder'] cmd += ['-r', '.', host + ':' + project_dir] run_local(cmd) for cmd in gene.config['ec2']['build_cmds']: run(options, cmd % environ, host, project_dir, gene) if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']: run( options, 'echo %s > %s' % ( gene.config['ec2']['apache_content'], gene.config['ec2']['apache_config'] ), host, project_dir, gene ) run(options, 'sudo apache2ctl graceful', host, project_dir, gene) # deploy if options.deploy_database and not options.local: _deploy(gene, host) if options.deploy_code or options.deploy_database \ or options.geodata and not options.local: # TODO not implemented yet create_snapshot(host, gene) if options.time: arguments = _get_arguments(options) arguments.extend(['--role', 'local']) arguments.extend(['--time', str(options.time)]) project_dir = None if options.local else gene.config['ec2']['code_folder'] processes = [] for i in range(gene.config['ec2']['number_process']): processes.append( run_remote_process( "%sgenerate_tiles %s" % ( _get_path(), ' '.join([str(a) for a in arguments]) ), host, project_dir, gene ) ) tiles_size = [] times = [] for p in processes: results = p.communicate() if results[1] != '': # pragma: no cover logger.debug('ERROR: %s' % results[1]) if PY3: results = [r.decode('utf-8') for r in results] results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), ) results = (re.sub(u'^[^\n]*\r', u'', results[0]), ) for r in results[0].split('\n'): if r.startswith('time: '): times.append(int(r.replace('time: ', ''))) elif r.startswith('size: '): tiles_size.append(int(r.replace('size: ', ''))) if len(times) == 0: # pragma: no cover logger.error("Not enough data") sys.exit(1) mean_time = reduce( lambda x, y: x + y, [timedelta(microseconds=int(r)) for r in times], timedelta() ) / len(times) ** 2 mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0 mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size) mean_size_kb = mean_size / 1024.0 print('==== Time results ====') print('A tile is generated in: %0.3f [ms]' % mean_time_ms) print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)) print('''config: cost: tileonly_generation_time: %0.3f tile_generation_time: %0.3f metatile_generation_time: 0 tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)) if options.shutdown: # pragma: no cover run(options, 'sudo shutdown 0', host, project_dir, gene) sys.exit(0) if options.fill_queue and not options.local: # pragma: no cover print("==== Till queue ====") # TODO test arguments = _get_arguments(options) arguments.extend(['--role', 'master', '--quiet']) project_dir = gene.config['ec2']['code_folder'] run_remote_process( options, "%sgenerate_tiles %s" % ( _get_path(), ' '.join([str(a) for a in arguments]) ), host, project_dir, gene ) sleep(5) attributes = gene.get_sqs_queue().get_attributes() print( "\rTiles to generate: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], ) ) if options.tiles_gen: # pragma: no cover print("==== Generate tiles ====") if options.wait and not options.local: print("") class Status(Thread): def run(self): # pragma: no cover while True: attributes = gene.get_sqs_queue().get_attributes() print( "\rTiles to generate/generating: %s/%s" % ( attributes['ApproximateNumberOfMessages'], attributes['ApproximateNumberOfMessagesNotVisible'], ) ) sleep(1) status_thread = Status() status_thread.setDaemon(True) status_thread.start() arguments = _get_arguments(options) arguments.extend(['--quiet']) if not options.local: arguments.extend(['--role', 'slave']) project_dir = None if options.local else gene.config['ec2']['code_folder'] threads = [] for i in range(gene.config['ec2']['number_process']): if options.local: threads.append(run_local_process( "%sgenerate_tiles --local-process-number %i %s" % ( _get_path(), i, ' '.join([str(a) for a in arguments]) ) )) else: run_remote_process( "%sgenerate_tiles %s" % ( _get_path(), ' '.join([str(a) for a in arguments]) ), host, project_dir, gene ) print('Tile generation started') if options.shutdown: run(options, 'sudo shutdown 0') if options.wait and options.local: while len(threads) > 0: threads = [t for t in threads if t.is_alive()] sleep(1) if 'sns' in gene.config: if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Host: %(host)s Command: %(cmd)s""" % { 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]) }, "Tile generation controller" )
def generate_resume(self, layer): if self._options.time is None: if layer is not None: all_dimensions = self._gene.get_all_dimensions(layer) message = [ "The tile generation of layer '{}{}' is finish".format( layer["name"], "" if ((len(all_dimensions) == 1 and len(all_dimensions[0]) == 0) or layer["type"] != "wms") else " ({})".format(" - ".join([ ", ".join( ["=".join(d) for d in dimensions.items()]) for dimensions in all_dimensions ])), ), ] else: message = ["The tile generation is finish"] if self._options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format( self._count_tiles.nb)) elif layer.get( "meta" ) if layer is not None else self._options.role == "slave": message += [ "Nb generated metatiles: {}".format( self._count_metatiles.nb), "Nb metatiles dropped: {}".format( self._count_metatiles_dropped.nb), ] if self._options.role != "master": message += [ "Nb generated tiles: {}".format(self._count_tiles.nb), "Nb tiles dropped: {}".format( self._count_tiles_dropped.nb), ] if self._options.role in ("local", "slave"): message += [ "Nb tiles stored: {}".format( self._count_tiles_stored.nb), "Nb tiles in error: {}".format(self._gene.error), "Total time: {}".format( duration_format(self._gene.duration)), ] if self._count_tiles_stored.nb != 0: message.append("Total size: {}".format( size_format(self._count_tiles_stored.size))) if self._count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (self._gene.duration / self._count_tiles.nb * 1000).seconds)) if self._count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( self._count_tiles_stored.size / self._count_tiles_stored.nb)) if not self._options.quiet and self._options.role in ( "local", "slave", "master") and message: print("\n".join(message) + "\n") if self._cache_tilestore is not None and hasattr( self._cache_tilestore, "connection"): self._cache_tilestore.connection.close() if (self._options.role != "hash" and self._options.time is None and "sns" in self._gene.config): # pragma: no cover if "region" in self._gene.config["sns"]: sns_client = boto3.client( "sns", region_name=self._gene.config["sns"]["region"]) else: sns_client = boto3.client("sns") sns_message = [message[0]] sns_message += [ "Layer: {}".format( layer["name"] if layer is not None else "(All layers)"), "Role: {}".format(self._options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(" ".join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] sns_client.publish( TopicArn=self._gene.config["sns"]["topic"], Message="\n".join(sns_message), Subject="Tile generation ({layer} - {role})".format( role=self._options.role, layer=layer["name"] if layer is not None else "All layers"), )
def generate_resume(self, layer): if self._options.time is None: if layer is not None: all_dimensions = self._gene.get_all_dimensions(layer) message = [ "The tile generation of layer '{}{}' is finish".format( layer['name'], "" if ( (len(all_dimensions) == 1 and len(all_dimensions[0]) == 0) or layer['type'] != 'wms' ) else " ({})".format( " - ".join([ ", ".join( ["=".join(d) for d in dimensions.items()] ) for dimensions in all_dimensions ]) ) ), ] else: message = [ "The tile generation is finish" ] if self._options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format(self._count_tiles.nb)) elif layer.get('meta') if layer is not None else self._options.role == "slave": message += [ "Nb generated metatiles: {}".format(self._count_metatiles.nb), "Nb metatiles dropped: {}".format(self._count_metatiles_dropped.nb), ] if self._options.role != "master": message += [ "Nb generated tiles: {}".format(self._count_tiles.nb), "Nb tiles dropped: {}".format(self._count_tiles_dropped.nb), ] if self._options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(self._count_tiles_stored.nb), "Nb tiles in error: {}".format(self._gene.error), "Total time: {}".format(duration_format(self._gene.duration)), ] if self._count_tiles_stored.nb != 0: message.append("Total size: {}".format(size_format(self._count_tiles_stored.size))) if self._count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (self._gene.duration / self._count_tiles.nb * 1000).seconds) ) if self._count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( self._count_tiles_stored.size / self._count_tiles_stored.nb) ) if not self._options.quiet and self._options.role in ('local', 'slave', 'master') and message: print("\n".join(message) + "\n") if self._cache_tilestore is not None and hasattr(self._cache_tilestore, 'connection'): self._cache_tilestore.connection.close() if self._options.role != 'hash' and \ self._options.time is None and \ 'sns' in self._gene.config: # pragma: no cover if 'region' in self._gene.config['sns']: sns_client = boto3.client('sns', region_name=self._gene.config['sns']['region']) else: sns_client = boto3.client('sns') sns_message = [message[0]] sns_message += [ "Layer: {}".format(layer['name'] if layer is not None else "(All layers)"), "Role: {}".format(self._options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] sns_client.publish(TopicArn=self._gene.config['sns']['topic'], Message="\n".join(sns_message), Subject="Tile generation ({layer} - {role})".format( role=self._options.role, layer=layer['name'] if layer is not None else "All layers"))
def gene(self, options, gene, layer): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() if options.role == 'slave' or options.get_hash or options.get_bbox: gene.layer = gene.layers[layer] else: gene.set_layer(layer, options) if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print \ "Tile bounds: [%i,%i,%i,%i]" % \ gene.layer['grid_ref']['obj'].extent(tilecoord) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e)) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore( gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords( [TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit("Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e)) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers']) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, sys.maxint)) for dim in gene.layer['dimensions']: params[dim['name']] = dim['value'] for dim in gene.options.dimensions: dim = dim.split('=') if len(dim) != 2: # pragma: no cover exit('the DIMENTIONS option should be like this ' 'DATE=2013 VERSION=13.') params[dim[0]] = dim[1] # Get the metatile image from the WMS server gene.get( URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=','.join(gene.layer['layers']), srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ), ), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': gene.get( MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene. layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get( MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find( "application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub('', tile.data)) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n**2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() self.count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap( HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") else: count_tiles = gene.counter(size=True) gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write( 'time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if not options.quiet and options.role in ('local', 'slave'): nb_tiles = count_tiles.nb + count_tiles_dropped.nb print """The tile generation of layer '%s' is finish %sNb generated tiles: %i Nb tiles dropped: %i Nb tiles stored: %i Nb error: %i Total time: %s Total size: %s Time per tiles: %i ms Size per tile: %i o """ % \ ( gene.layer['name'], """Nb generated metatiles: %i Nb metatiles dropped: %i """ % ( count_metatiles.nb, count_metatiles_dropped.nb ) if meta else '', nb_tiles, count_tiles_dropped.nb, count_tiles.nb, gene.error, duration_format(gene.duration), size_format(count_tiles.size), (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1 ) if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region( gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Layer: %(layer)s Role: %(role)s Host: %(host)s Command: %(cmd)s %(meta)sNb generated tiles: %(nb_tiles)i Nb tiles dropped: %(nb_tiles_dropped)i Total time: %(duration)s [s] Time per tiles: %(tile_duration)i [ms]""" % { 'role': options.role, 'layer': gene.layer['name'], 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]), 'meta': """Nb generated metatiles: %(nb_metatiles)i Nb metatiles dropped: %(nb_metatiles_dropped)i """ % { 'nb_metatiles': count_metatiles.nb, 'nb_metatiles_dropped': count_metatiles_dropped.nb, } if meta else '', 'nb_tiles': nb_tiles if meta else count_metatiles.nb, 'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb, 'duration': duration_format(gene.duration), 'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, }, "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] })
def _gene(self, options, gene, layer, dimensions={}): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() count_tiles_stored = None if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print( "Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord) ) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e) ) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore(gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache, dimensions) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e) ) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) count_tiles = gene.counter() elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers'].split(',')) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, 999999)) params.update(dimensions) # Get the metatile image from the WMS server gene.get(URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=gene.layer['layers'], srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ),), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': # pragma: no cover from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': count_tiles = gene.counter() gene.get(MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get(MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find("application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub( '', tile.data.decode('utf-8') if PY3 else tile.data ) ) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data ) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n ** 2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() if gene.layer['type'] != 'mapnik' or gene.layer['output_format'] != 'grid': count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: # pragma: no cover gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() else: # pragma: no cover count_tiles = gene.counter() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles_stored = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) message = [] if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() message = [ "The tile generation of layer '{}{}' is finish".format( gene.layer['name'], "" if len(dimensions) == 0 or gene.layer['type'] != 'wms' else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()]) ), ] if options.role == "master": # pragma: no cover message.append("Nb of generated jobs: {}".format(count_tiles.nb)) else: if meta: message += [ "Nb generated metatiles: {}".format(count_metatiles.nb), "Nb metatiles dropped: {}".format(count_metatiles_dropped.nb), ] message += [ "Nb generated tiles: {}".format(count_tiles.nb), "Nb tiles dropped: {}".format(count_tiles_dropped.nb), ] if options.role in ('local', 'slave'): message += [ "Nb tiles stored: {}".format(count_tiles_stored.nb), "Nb tiles in error: {}".format(gene.error), "Total time: {}".format(duration_format(gene.duration)), ] if count_tiles_stored.nb != 0: message.append("Total size: {}".format(size_format(count_tiles_stored.size))) if count_tiles.nb != 0: message.append("Time per tile: {:0.0f} ms".format( (gene.duration / count_tiles.nb * 1000).seconds) ) if count_tiles_stored.nb != 0: message.append("Size per tile: {:0.0f} o".format( count_tiles_stored.size / count_tiles_stored.nb) ) if not options.quiet and options.role in ('local', 'slave'): print("\n".join(message) + "\n") if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() sns_message = [message[0]] sns_message += [ "Layer: {}".format(gene.layer['name']), "Role: {}".format(options.role), "Host: {}".format(socket.getfqdn()), "Command: {}".format(' '.join([quote(arg) for arg in sys.argv])), ] sns_message += message[1:] connection.publish( gene.config['sns']['topic'], "\n".join(sns_message), "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] } )
def gene(self, options, gene, layer): count_metatiles = None count_metatiles_dropped = Count() count_tiles = None count_tiles_dropped = Count() if options.role == 'slave' or options.get_hash or options.get_bbox: gene.layer = gene.layers[layer] else: gene.set_layer(layer, options) if options.get_bbox: try: tilecoord = parse_tilecoord(options.get_bbox) print( "Tile bounds: [%i,%i,%i,%i]" % gene.layer['grid_ref']['obj'].extent(tilecoord) ) exit() except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" % (options.get_bbox, e) ) if options.get_hash: options.role = 'hash' options.test = 1 sqs_tilestore = None if options.role in ('master', 'slave'): # Create SQS queue sqs_tilestore = SQSTileStore(gene.get_sqs_queue()) # pragma: no cover cache_tilestore = None if options.role in ('local', 'slave'): cache_tilestore = gene.get_tilesstore(options.cache) meta = gene.layer['meta'] if options.tiles: gene.set_store(TilesFileStore(options.tiles)) elif options.role in ('local', 'master'): # Generate a stream of metatiles gene.init_tilecoords() gene.add_geom_filter() if options.local_process_number is not None: # pragma: no cover gene.add_local_process_filter() elif options.role == 'slave': # Get the metatiles from the SQS queue gene.set_store(sqs_tilestore) # pragma: no cover elif options.role == 'hash': try: z, x, y = (int(v) for v in options.get_hash.split('/')) if meta: gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])]) else: gene.set_tilecoords([TileCoord(z, x, y)]) except ValueError as e: # pragma: no cover exit( "Tile '%s' is not in the format 'z/x/y'\n%r" % (options.get_hash, e) ) # At this stage, the tilestream contains metatiles that intersect geometry gene.add_logger() count_metatiles = gene.counter() if options.role == 'master': # pragma: no cover # Put the metatiles into the SQS queue gene.put(sqs_tilestore) elif options.role in ('local', 'slave', 'hash'): if gene.layer['type'] == 'wms': params = gene.layer['params'].copy() if 'STYLES' not in params: params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers']) if gene.layer['generate_salt']: params['SALT'] = str(random.randint(0, sys.maxint)) for dim in gene.layer['dimensions']: params[dim['name']] = dim['value'] for dim in gene.options.dimensions: dim = dim.split('=') if len(dim) != 2: # pragma: no cover exit( 'the DIMENTIONS option should be like this ' 'DATE=2013 VERSION=13.' ) params[dim[0]] = dim[1] # Get the metatile image from the WMS server gene.get(URLTileStore( tilelayouts=(WMSTileLayout( url=gene.layer['url'], layers=','.join(gene.layer['layers']), srs=gene.layer['grid_ref']['srs'], format=gene.layer['mime_type'], border=gene.layer['meta_buffer'] if meta else 0, tilegrid=gene.get_grid()['obj'], params=params, ),), headers=gene.layer['headers'], ), "Get tile from WMS") elif gene.layer['type'] == 'mapnik': from tilecloud.store.mapnik_ import MapnikTileStore from tilecloud_chain.mapnik_ import MapnikDropActionTileStore grid = gene.get_grid() if gene.layer['output_format'] == 'grid': gene.get(MapnikDropActionTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], resolution=gene.layer['resolution'], layers_fields=gene.layer['layers_fields'], drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, proj4_literal=grid['proj4_literal'], ), "Create Mapnik grid tile") else: gene.get(MapnikTileStore( tilegrid=grid['obj'], mapfile=gene.layer['mapfile'], image_buffer=gene.layer['meta_buffer'] if meta else 0, data_buffer=gene.layer['data_buffer'], output_format=gene.layer['output_format'], proj4_literal=grid['proj4_literal'], ), "Create Mapnik tile") def wrong_content_type_to_error(tile): if tile is not None and tile.content_type is not None \ and tile.content_type.find("image/") != 0: if tile.content_type.find("application/vnd.ogc.se_xml") == 0: tile.error = "WMS server error: %s" % ( self._re_rm_xml_tag.sub('', tile.data) ) else: # pragma: no cover tile.error = "%s is not an image format, error: %s" % ( tile.content_type, tile.data ) return tile gene.imap(wrong_content_type_to_error) # Handle errors gene.add_error_filters() if meta: if options.role == 'hash': gene.imap(HashLogger('empty_metatile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_metatile_detection' in gene.layer: empty_tile = gene.layer['empty_metatile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_metatiles_dropped, )) def add_elapsed_togenerate(metatile): if metatile is not None: metatile.elapsed_togenerate = metatile.tilecoord.n ** 2 return True return False # pragma: no cover gene.ifilter(add_elapsed_togenerate) # Split the metatile image into individual tiles gene.add_metatile_splitter() gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s')) # Handle errors gene.add_error_filters() self.count_tiles = gene.counter() if 'pre_hash_post_process' in gene.layer: gene.process(gene.layer['pre_hash_post_process']) if options.role == 'hash': gene.imap(HashLogger('empty_tile_detection')) elif not options.near: # Discard tiles with certain content if 'empty_tile_detection' in gene.layer: empty_tile = gene.layer['empty_tile_detection'] gene.imap(HashDropper( empty_tile['size'], empty_tile['hash'], store=cache_tilestore, queue_store=sqs_tilestore, count=count_tiles_dropped, )) gene.process() if options.role in ('local', 'slave'): gene.add_error_filters() gene.ifilter(DropEmpty(gene)) count_tiles = gene.counter(size=True) if options.time: def log_size(tile): sys.stdout.write('size: %i\n' % len(tile.data)) return tile gene.imap(log_size) gene.put(cache_tilestore, "Store the tile") else: count_tiles = gene.counter(size=True) gene.add_error_filters() if options.generated_tiles_file: # pragma: no cover generated_tiles_file = open(options.generated_tiles_file, 'a') def do(tile): generated_tiles_file.write('%s\n' % (tile.tilecoord, )) return tile gene.imap(do) if options.role == 'slave': # pragma: no cover if meta: def decr_tile_in_metatile(tile): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0: sqs_tilestore.delete_one(tile.metatile) return True gene.ifilter(decr_tile_in_metatile) else: gene.delete(sqs_tilestore) if options.time is not None: class LogTime: n = 0 t1 = None def __call__(self, tile): self.n += 1 if self.n == options.time: self.t1 = datetime.now() elif self.n == 2 * options.time: t2 = datetime.now() d = (t2 - self.t1) / options.time sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds)) return tile gene.imap(LogTime()) gene.consume(options.time * 3) else: gene.consume() if not options.quiet and options.role in ('local', 'slave'): nb_tiles = count_tiles.nb + count_tiles_dropped.nb print( """The tile generation of layer '%s' is finish %sNb generated tiles: %i Nb tiles dropped: %i Nb tiles stored: %i Nb error: %i Total time: %s Total size: %s Time per tiles: %i ms Size per tile: %i o """ % ( gene.layer['name'], """Nb generated metatiles: %i Nb metatiles dropped: %i """ % ( count_metatiles.nb, count_metatiles_dropped.nb ) if meta else '', nb_tiles, count_tiles_dropped.nb, count_tiles.nb, gene.error, duration_format(gene.duration), size_format(count_tiles.size), (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1 ) ) if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'): cache_tilestore.connection.close() if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover if 'region' in gene.config['sns']: connection = sns.connect_to_region(gene.config['sns']['region']) else: connection = boto.connect_sns() connection.publish( gene.config['sns']['topic'], """The tile generation is finish Layer: %(layer)s Role: %(role)s Host: %(host)s Command: %(cmd)s %(meta)sNb generated tiles: %(nb_tiles)i Nb tiles dropped: %(nb_tiles_dropped)i Total time: %(duration)s [s] Time per tiles: %(tile_duration)i [ms]""" % { 'role': options.role, 'layer': gene.layer['name'], 'host': socket.getfqdn(), 'cmd': ' '.join([quote(arg) for arg in sys.argv]), 'meta': """Nb generated metatiles: %(nb_metatiles)i Nb metatiles dropped: %(nb_metatiles_dropped)i """ % { 'nb_metatiles': count_metatiles.nb, 'nb_metatiles_dropped': count_metatiles_dropped.nb, } if meta else '', 'nb_tiles': nb_tiles if meta else count_metatiles.nb, 'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb, 'duration': duration_format(gene.duration), 'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0, }, "Tile generation (%(layer)s - %(role)s)" % { 'role': options.role, 'layer': gene.layer['name'] } )