def run_local_process(cmd):
if type(cmd) != list:
cmd = cmd.split(' ')
logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd]))
task = Run(cmd)
task.start()
return task
def run_local_process(cmd):
if type(cmd) != list:
cmd = cmd.split(' ')
logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd]))
task = Run(cmd)
task.start()
return task
def run_remote_process(remote_cmd, host, project_dir, gene):
cmd = ['ssh']
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd.extend(gene.config['ec2']['ssh_options'].split(' '))
if host is None: # pragma: no cover
exit('host option is required.')
cmd.append(host)
env = ''
if os.getenv('AWS_ACCESS_KEY_ID') and os.getenv('AWS_SECRET_ACCESS_KEY'): # pragma: no cover
env = 'export AWS_ACCESS_KEY_ID=%(access_key)s;export AWS_SECRET_ACCESS_KEY=%(secret_key)s;' % {
'access_key': os.getenv('AWS_ACCESS_KEY_ID'),
'secret_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
}
cmd.append(
'cd %(project_dir)s;'
'%(env)s'
'%(cmd)s' % {
'cmd': remote_cmd,
'env': env,
'project_dir': project_dir
}
)
logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd]))
return Popen(cmd, stdout=PIPE, stderr=PIPE)
def run_local(cmd):
if type(cmd) != list:
cmd = cmd.split(' ')
logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd]))
result = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()
logger.info(result[0])
logger.error(result[1])
return result
def run_local(cmd):
if type(cmd) != list:
cmd = cmd.split(' ')
logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd]))
result = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()
logger.info(result[0])
logger.error(result[1])
return result
def run_local(cmd):
if type(cmd) != list:
cmd = cmd.split(' ')
logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd]))
result = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()
if PY3:
result = [r.decode('utf-8') for r in result]
if len(result[0]) != 0: # pragma: no cover
logger.info(result[0])
if len(result[1]) != 0:
logger.error(result[1])
return result
def run_local(cmd):
if type(cmd) != list:
cmd = cmd.split(' ')
logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd]))
result = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()
if PY3:
result = [r.decode('utf-8') for r in result]
if len(result[0]) != 0: # pragma: no cover
logger.info(result[0])
if len(result[1]) != 0:
logger.error(result[1])
return result
def run_remote_process(remote_cmd, host, project_dir, gene):
cmd = ['ssh']
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd.extend(gene.config['ec2']['ssh_options'].split(' '))
if host is None: # pragma: no cover
exit('host option is required.')
cmd.append(host)
env = ''
if os.getenv('AWS_ACCESS_KEY_ID') and os.getenv(
'AWS_SECRET_ACCESS_KEY'): # pragma: no cover
env = 'export AWS_ACCESS_KEY_ID=%(access_key)s;export AWS_SECRET_ACCESS_KEY=%(secret_key)s;' % {
'access_key': os.getenv('AWS_ACCESS_KEY_ID'),
'secret_key': os.getenv('AWS_SECRET_ACCESS_KEY'),
}
cmd.append('cd %(project_dir)s;'
'%(env)s'
'%(cmd)s' % {
'cmd': remote_cmd,
'env': env,
'project_dir': project_dir
})
logger.debug('Run: %s.' % ' '.join([quote(c) for c in cmd]))
return Popen(cmd, stdout=PIPE, stderr=PIPE)
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog=sys.argv[0])
add_comon_options(parser)
parser.add_argument('--deploy-config',
default=None,
dest="deploy_config",
metavar="FILE",
help='path to the deploy configuration file')
parser.add_argument('--status',
default=False,
action="store_true",
help='display the SQS queue status and exit')
parser.add_argument('--disable-geodata',
default=True,
action="store_false",
dest="geodata",
help='disable geodata synchronisation')
parser.add_argument('--disable-code',
default=True,
action="store_false",
dest="deploy_code",
help='disable deploy application code')
parser.add_argument('--disable-database',
default=True,
action="store_false",
dest="deploy_database",
help='disable deploy database')
parser.add_argument('--disable-fillqueue',
default=True,
action="store_false",
dest="fill_queue",
help='disable queue filling')
parser.add_argument('--disable-tilesgen',
default=True,
action="store_false",
dest="tiles_gen",
help='disable tile generation')
parser.add_argument('--host',
default=None,
help='The host used to generate tiles')
parser.add_argument('--shutdown',
default=False,
action="store_true",
help='Shut done the remote host after the task.')
parser.add_argument('--wait',
default=False,
action="store_true",
help='Wait that all the tasks will finish.')
parser.add_argument('--local',
default=False,
action="store_true",
help='Run the generation locally')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print("EC2 not configured")
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if not options.local and options.geodata and 'geodata_folder' in gene.config[
'ec2']: # pragma: no cover
print("==== Sync geodata ====")
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code and not options.local:
print("==== Sync and build code ====")
cmd = [
'rsync',
'--delete',
]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run(options, cmd % environ, host, project_dir, gene)
if 'apache_content' in gene.config[
'ec2'] and 'apache_config' in gene.config['ec2']:
run(
options,
'echo %s > %s' % (gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']), host,
project_dir, gene)
run(options, 'sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database and not options.local:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata and not options.local:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = None if options.local else gene.config['ec2'][
'code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
"%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene))
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
if PY3:
results = [r.decode('utf-8') for r in results]
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(lambda x, y: x + y,
[timedelta(microseconds=int(r))
for r in times], timedelta()) / len(times)**2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print('==== Time results ====')
print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
print('''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))
if options.shutdown: # pragma: no cover
run(options, 'sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue and not options.local: # pragma: no cover
print("==== Till queue ====")
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master', '--quiet'])
project_dir = gene.config['ec2']['code_folder']
run_remote_process(
options, "%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene)
sleep(5)
attributes = gene.get_sqs_queue().get_attributes()
print("\rTiles to generate: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
))
if options.tiles_gen: # pragma: no cover
print("==== Generate tiles ====")
if options.wait and not options.local:
print("")
class Status(Thread):
def run(self): # pragma: no cover
while True:
attributes = gene.get_sqs_queue().get_attributes()
print("\rTiles to generate/generating: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
))
sleep(1)
status_thread = Status()
status_thread.setDaemon(True)
status_thread.start()
arguments = _get_arguments(options)
arguments.extend(['--quiet'])
if not options.local:
arguments.extend(['--role', 'slave'])
project_dir = None if options.local else gene.config['ec2'][
'code_folder']
threads = []
for i in range(gene.config['ec2']['number_process']):
if options.local:
threads.append(
run_local_process(
"%sgenerate_tiles --local-process-number %i %s" %
(_get_path(), i, ' '.join([str(a)
for a in arguments]))))
else:
run_remote_process(
"%sgenerate_tiles %s" %
(_get_path(), ' '.join([str(a) for a in arguments])), host,
project_dir, gene)
print('Tile generation started')
if options.shutdown:
run(options, 'sudo shutdown 0')
if options.wait and options.local:
while len(threads) > 0:
threads = [t for t in threads if t.is_alive()]
sleep(1)
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
}, "Tile generation controller")
def _gene(self, options, gene, layer, dimensions={}):
count_metatiles = None
count_metatiles_dropped = Count()
count_tiles = None
count_tiles_dropped = Count()
count_tiles_stored = None
if options.get_bbox:
try:
tilecoord = parse_tilecoord(options.get_bbox)
print("Tile bounds: [%i,%i,%i,%i]" %
gene.layer['grid_ref']['obj'].extent(tilecoord))
exit()
except ValueError as e: # pragma: no cover
exit(
"Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r"
% (options.get_bbox, e))
if options.get_hash:
options.role = 'hash'
options.test = 1
sqs_tilestore = None
if options.role in ('master', 'slave'):
# Create SQS queue
sqs_tilestore = SQSTileStore(
gene.get_sqs_queue()) # pragma: no cover
cache_tilestore = None
if options.role in ('local', 'slave'):
cache_tilestore = gene.get_tilesstore(options.cache, dimensions)
meta = gene.layer['meta']
if options.tiles:
gene.set_store(TilesFileStore(options.tiles))
elif options.role in ('local', 'master'):
# Generate a stream of metatiles
gene.init_tilecoords()
gene.add_geom_filter()
if options.local_process_number is not None: # pragma: no cover
gene.add_local_process_filter()
elif options.role == 'slave':
# Get the metatiles from the SQS queue
gene.set_store(sqs_tilestore) # pragma: no cover
elif options.role == 'hash':
try:
z, x, y = (int(v) for v in options.get_hash.split('/'))
if meta:
gene.set_tilecoords(
[TileCoord(z, x, y, gene.layer['meta_size'])])
else:
gene.set_tilecoords([TileCoord(z, x, y)])
except ValueError as e: # pragma: no cover
exit("Tile '%s' is not in the format 'z/x/y'\n%r" %
(options.get_hash, e))
# At this stage, the tilestream contains metatiles that intersect geometry
gene.add_logger()
count_metatiles = gene.counter()
if options.role == 'master': # pragma: no cover
# Put the metatiles into the SQS queue
gene.put(sqs_tilestore)
count_tiles = gene.counter()
elif options.role in ('local', 'slave', 'hash'):
if gene.layer['type'] == 'wms':
params = gene.layer['params'].copy()
if 'STYLES' not in params:
params['STYLES'] = ','.join(
gene.layer['wmts_style']
for l in gene.layer['layers'].split(','))
if gene.layer['generate_salt']:
params['SALT'] = str(random.randint(0, 999999))
params.update(dimensions)
# Get the metatile image from the WMS server
gene.get(
URLTileStore(
tilelayouts=(WMSTileLayout(
url=gene.layer['url'],
layers=gene.layer['layers'],
srs=gene.layer['grid_ref']['srs'],
format=gene.layer['mime_type'],
border=gene.layer['meta_buffer'] if meta else 0,
tilegrid=gene.get_grid()['obj'],
params=params,
), ),
headers=gene.layer['headers'],
), "Get tile from WMS")
elif gene.layer['type'] == 'mapnik': # pragma: no cover
from tilecloud.store.mapnik_ import MapnikTileStore
from tilecloud_chain.mapnik_ import MapnikDropActionTileStore
grid = gene.get_grid()
if gene.layer['output_format'] == 'grid':
count_tiles = gene.counter()
gene.get(
MapnikDropActionTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer']
if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
resolution=gene.layer['resolution'],
layers_fields=gene.layer['layers_fields'],
drop_empty_utfgrid=gene.
layer['drop_empty_utfgrid'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
proj4_literal=grid['proj4_literal'],
), "Create Mapnik grid tile")
else:
gene.get(
MapnikTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer']
if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
proj4_literal=grid['proj4_literal'],
), "Create Mapnik tile")
def wrong_content_type_to_error(tile):
if tile is not None and tile.content_type is not None \
and tile.content_type.find("image/") != 0:
if tile.content_type.find(
"application/vnd.ogc.se_xml") == 0:
tile.error = "WMS server error: %s" % (
self._re_rm_xml_tag.sub(
'',
tile.data.decode('utf-8')
if PY3 else tile.data))
else: # pragma: no cover
tile.error = "%s is not an image format, error: %s" % (
tile.content_type, tile.data)
return tile
gene.imap(wrong_content_type_to_error)
# Handle errors
gene.add_error_filters()
if meta:
if options.role == 'hash':
gene.imap(HashLogger('empty_metatile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_metatile_detection' in gene.layer:
empty_tile = gene.layer['empty_metatile_detection']
gene.imap(
HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_metatiles_dropped,
))
def add_elapsed_togenerate(metatile):
if metatile is not None:
metatile.elapsed_togenerate = metatile.tilecoord.n**2
return True
return False # pragma: no cover
gene.ifilter(add_elapsed_togenerate)
# Split the metatile image into individual tiles
gene.add_metatile_splitter()
gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))
# Handle errors
gene.add_error_filters()
if gene.layer['type'] != 'mapnik' or gene.layer[
'output_format'] != 'grid':
count_tiles = gene.counter()
if 'pre_hash_post_process' in gene.layer: # pragma: no cover
gene.process(gene.layer['pre_hash_post_process'])
if options.role == 'hash':
gene.imap(HashLogger('empty_tile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_tile_detection' in gene.layer:
empty_tile = gene.layer['empty_tile_detection']
gene.imap(
HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
))
gene.process()
else: # pragma: no cover
count_tiles = gene.counter()
if options.role in ('local', 'slave'):
gene.add_error_filters()
gene.ifilter(DropEmpty(gene))
count_tiles_stored = gene.counter(size=True)
if options.time:
def log_size(tile):
sys.stdout.write('size: %i\n' % len(tile.data))
return tile
gene.imap(log_size)
gene.put(cache_tilestore, "Store the tile")
gene.add_error_filters()
if options.generated_tiles_file: # pragma: no cover
generated_tiles_file = open(options.generated_tiles_file, 'a')
def do(tile):
generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
return tile
gene.imap(do)
if options.role == 'slave': # pragma: no cover
if meta:
def decr_tile_in_metatile(tile):
tile.metatile.elapsed_togenerate -= 1
if tile.metatile.elapsed_togenerate == 0:
sqs_tilestore.delete_one(tile.metatile)
return True
gene.ifilter(decr_tile_in_metatile)
else:
gene.delete(sqs_tilestore)
message = []
if options.time is not None:
class LogTime:
n = 0
t1 = None
def __call__(self, tile):
self.n += 1
if self.n == options.time:
self.t1 = datetime.now()
elif self.n == 2 * options.time:
t2 = datetime.now()
d = (t2 - self.t1) / options.time
sys.stdout.write(
'time: %i\n' %
((d.days * 24 * 3600 + d.seconds) * 1000000 +
d.microseconds))
return tile
gene.imap(LogTime())
gene.consume(options.time * 3)
else:
gene.consume()
message = [
"The tile generation of layer '{}{}' is finish".format(
gene.layer['name'], "" if len(dimensions) == 0
or gene.layer['type'] != 'wms' else " (%s)" %
", ".join(["=".join(d) for d in dimensions.items()])),
]
if options.role == "master": # pragma: no cover
message.append("Nb of generated jobs: {}".format(
count_tiles.nb))
else:
if meta:
message += [
"Nb generated metatiles: {}".format(
count_metatiles.nb),
"Nb metatiles dropped: {}".format(
count_metatiles_dropped.nb),
]
message += [
"Nb generated tiles: {}".format(count_tiles.nb),
"Nb tiles dropped: {}".format(count_tiles_dropped.nb),
]
if options.role in ('local', 'slave'):
message += [
"Nb tiles stored: {}".format(count_tiles_stored.nb),
"Nb tiles in error: {}".format(gene.error),
"Total time: {}".format(duration_format(
gene.duration)),
]
if count_tiles_stored.nb != 0:
message.append("Total size: {}".format(
size_format(count_tiles_stored.size)))
if count_tiles.nb != 0:
message.append("Time per tile: {:0.0f} ms".format(
(gene.duration / count_tiles.nb * 1000).seconds))
if count_tiles_stored.nb != 0:
message.append("Size per tile: {:0.0f} o".format(
count_tiles_stored.size / count_tiles_stored.nb))
if not options.quiet and options.role in ('local', 'slave'):
print("\n".join(message) + "\n")
if cache_tilestore is not None and hasattr(cache_tilestore,
'connection'):
cache_tilestore.connection.close()
if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
sns_message = [message[0]]
sns_message += [
"Layer: {}".format(gene.layer['name']),
"Role: {}".format(options.role),
"Host: {}".format(socket.getfqdn()),
"Command: {}".format(' '.join([quote(arg)
for arg in sys.argv])),
]
sns_message += message[1:]
connection.publish(
gene.config['sns']['topic'], "\n".join(sns_message),
"Tile generation (%(layer)s - %(role)s)" % {
'role': options.role,
'layer': gene.layer['name']
})
def _gene(self, options, gene, dimensions=None):
if dimensions is None: # pragma: no cover
dimensions = {}
self.dimensions = dimensions
self.count_metatiles = None
self.count_metatiles_dropped = Count()
self.count_tiles = Count()
self.count_tiles_dropped = Count()
self.count_tiles_stored = None
self.sqs_tilestore = None
self.cache_tilestore = None
if options.get_bbox:
try:
tilecoord = parse_tilecoord(options.get_bbox)
print("Tile bounds: [{},{},{},{}]".format(*default_int(
gene.layer['grid_ref']['obj'].extent(tilecoord))))
exit()
except ValueError as e: # pragma: no cover
print(
"Tile '{}' is not in the format 'z/x/y' or z/x/y:+n/+n\n{}"
.format(options.get_bbox, repr(e)))
exit(1)
if options.get_hash:
options.role = 'hash'
options.test = 1
if options.role in ('master', 'slave'):
# Create SQS queue
self.sqs_tilestore = SQSTileStore(
gene.get_sqs_queue(),
on_empty=await_message
if options.daemon else maybe_stop) # pragma: no cover
if options.role in ('local', 'slave'):
self.cache_tilestore = gene.get_tilesstore(options.cache,
dimensions)
if options.tiles:
gene.set_store(TilesFileStore(options.tiles, options.layer))
elif options.role in ('local', 'master'):
# Generate a stream of metatiles
gene.init_tilecoords()
gene.add_geom_filter()
if options.role in ('local', 'master') and 'logging' in gene.config:
gene.imap(
DatabaseLoggerInit(gene.config['logging'], options is not None
and options.daemon))
if options.local_process_number is not None: # pragma: no cover
gene.add_local_process_filter()
elif options.role == 'slave':
# Get the metatiles from the SQS queue
gene.set_store(self.sqs_tilestore) # pragma: no cover
elif options.role == 'hash':
try:
z, x, y = (int(v) for v in options.get_hash.split('/'))
if gene.layer.get('meta'):
gene.set_tilecoords(
[TileCoord(z, x, y, gene.layer['meta_size'])])
else:
gene.set_tilecoords([TileCoord(z, x, y)])
except ValueError as e: # pragma: no cover
exit("Tile '{}' is not in the format 'z/x/y'\n{}".format(
options.get_hash, repr(e)))
# At this stage, the tilestream contains metatiles that intersect geometry
gene.add_logger()
self.count_metatiles = gene.counter()
if options.role == 'master': # pragma: no cover
# Put the metatiles into the SQS queue
gene.put(self.sqs_tilestore)
self.count_tiles = gene.counter()
elif options.role in ('local', 'slave', 'hash'):
gene.get(
MultiTileStore({
name: self._get_tilestore_for_layer(layer, gene)
for name, layer in gene.layers.items()
}), 'Get tile')
def wrong_content_type_to_error(tile):
if tile is not None and tile.content_type is not None \
and tile.content_type.find("image/") != 0:
if tile.content_type.find(
"application/vnd.ogc.se_xml") == 0:
tile.error = "WMS server error: {}".format(
(self._re_rm_xml_tag.sub('', tile.error)))
else: # pragma: no cover
tile.error = "{} is not an image format, error: {}".format(
tile.content_type, tile.error)
return tile
gene.imap(wrong_content_type_to_error)
gene.add_error_filters()
if options.role == 'hash':
if gene.layer.get('meta', False):
gene.imap(HashLogger('empty_metatile_detection'))
elif not options.near:
droppers = {}
for lname, layer in gene.layers.items():
if 'empty_metatile_detection' in layer:
empty_tile = layer['empty_metatile_detection']
droppers[lname] = HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=self.cache_tilestore,
queue_store=self.sqs_tilestore,
count=self.count_metatiles_dropped,
)
if droppers:
gene.imap(MultiAction(droppers))
def add_elapsed_togenerate(metatile):
if metatile is not None:
metatile.elapsed_togenerate = metatile.tilecoord.n**2
return True
return False # pragma: no cover
gene.ifilter(add_elapsed_togenerate)
# Split the metatile image into individual tiles
gene.add_metatile_splitter()
gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))
gene.imap(self.count_tiles)
gene.process(key='pre_hash_post_process')
if options.role == 'hash':
gene.imap(HashLogger('empty_tile_detection'))
elif not options.near:
droppers = {}
for lname, layer in gene.layers.items():
if 'empty_tile_detection' in layer:
empty_tile = layer['empty_tile_detection']
droppers[lname] = HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=self.cache_tilestore,
queue_store=self.sqs_tilestore,
count=self.count_tiles_dropped,
)
if droppers:
gene.imap(MultiAction(droppers))
gene.process()
else: # pragma: no cover
self.count_tiles = gene.counter()
if options.role in ('local', 'slave'):
self.count_tiles_stored = gene.counter(size=True)
if options.time:
def log_size(tile):
sys.stdout.write('size: {}\n'.format(len(tile.data)))
return tile
gene.imap(log_size)
gene.put(self.cache_tilestore, "Store the tile")
if options.generated_tiles_file: # pragma: no cover
generated_tiles_file = open(options.generated_tiles_file, 'a')
def do(tile):
generated_tiles_file.write('{}\n'.format(tile.tilecoord))
return tile
gene.imap(do)
if options.role == 'slave': # pragma: no cover
def delete_from_store(tile):
if hasattr(tile, 'metatile'):
tile.metatile.elapsed_togenerate -= 1
if tile.metatile.elapsed_togenerate == 0:
self.sqs_tilestore.delete_one(tile.metatile)
else:
self.sqs_tilestore.delete_one(tile)
return True
gene.ifilter(delete_from_store)
if options.role in ('local', 'slave') and 'logging' in gene.config:
gene.imap(
DatabaseLogger(gene.config['logging'], options is not None
and options.daemon))
gene.add_error_filters()
message = []
if options.time is not None:
class LogTime:
n = 0
t1 = None
def __call__(self, tile):
self.n += 1
if self.n == options.time:
self.t1 = datetime.now()
elif self.n == 2 * options.time:
t2 = datetime.now()
d = (t2 - self.t1) / options.time
sys.stdout.write('time: {}\n'.format(
((d.days * 24 * 3600 + d.seconds) * 1000000 +
d.microseconds)))
return tile
gene.imap(LogTime())
gene.consume(options.time * 3)
else:
gene.consume()
if gene.layer is not None:
message = [
"The tile generation of layer '{}{}' is finish".format(
gene.layer['name'], ""
if len(dimensions) == 0 or gene.layer['type'] != 'wms'
else " ({})".format(", ".join(
["=".join(d) for d in dimensions.items()]))),
]
if options.role == "master": # pragma: no cover
message.append("Nb of generated jobs: {}".format(
self.count_tiles.nb))
else:
if gene.layer.get('meta'):
message += [
"Nb generated metatiles: {}".format(
self.count_metatiles.nb),
"Nb metatiles dropped: {}".format(
self.count_metatiles_dropped.nb),
]
else:
message = ["The tile generation is finish"]
if options.role != "master":
message += [
"Nb generated tiles: {}".format(self.count_tiles.nb),
"Nb tiles dropped: {}".format(self.count_tiles_dropped.nb),
]
if options.role in ('local', 'slave'):
message += [
"Nb tiles stored: {}".format(
self.count_tiles_stored.nb),
"Nb tiles in error: {}".format(gene.error),
"Total time: {}".format(duration_format(
gene.duration)),
]
if self.count_tiles_stored.nb != 0:
message.append("Total size: {}".format(
size_format(self.count_tiles_stored.size)))
if self.count_tiles.nb != 0:
message.append("Time per tile: {:0.0f} ms".format(
(gene.duration / self.count_tiles.nb *
1000).seconds))
if self.count_tiles_stored.nb != 0:
message.append("Size per tile: {:0.0f} o".format(
self.count_tiles_stored.size /
self.count_tiles_stored.nb))
if not options.quiet and options.role in ('local', 'slave'):
print("\n".join(message) + "\n")
if self.cache_tilestore is not None and hasattr(
self.cache_tilestore, 'connection'):
self.cache_tilestore.connection.close()
if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
sns_message = [message[0]]
sns_message += [
"Layer: {}".format(gene.layer['name'] if gene.
layer is not None else "(All layers)"),
"Role: {}".format(options.role),
"Host: {}".format(socket.getfqdn()),
"Command: {}".format(' '.join([quote(arg)
for arg in sys.argv])),
]
sns_message += message[1:]
connection.publish(
gene.config['sns']['topic'], "\n".join(sns_message),
"Tile generation ({layer!s} - {role!s})".format(
**{
'role':
options.role,
'layer':
gene.layer['name'] if gene.
layer is not None else "All layers"
}))
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog='./buildout/bin/generate_amazon'
)
add_comon_options(parser)
parser.add_argument(
'--deploy-config', default=None, dest="deploy_config", metavar="FILE",
help='path to the deploy configuration file'
)
parser.add_argument(
'--status', default=False, action="store_true",
help='display the SQS queue status and exit'
)
parser.add_argument(
'--disable-geodata', default=True, action="store_false", dest="geodata",
help='disable geodata synchronisation'
)
parser.add_argument(
'--disable-code', default=True, action="store_false", dest="deploy_code",
help='disable deploy application code'
)
parser.add_argument(
'--disable-database', default=True, action="store_false", dest="deploy_database",
help='disable deploy database'
)
parser.add_argument(
'--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
help='disable queue filling'
)
parser.add_argument(
'--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
help='disable tile generation'
)
parser.add_argument(
'--host', default=None,
help='The host used to generate tiles'
)
parser.add_argument(
'--shutdown', default=False, action="store_true",
help='Shut done the remote host after the task.'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print "EC2 not configured"
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if options.geodata and 'geodata_folder' in gene.config['ec2']:
print "==== Sync geodata ===="
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options,
'-r', gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code:
print "==== Sync and build code ===="
error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
if error:
exit(1) # pragma: no cover
cmd = ['rsync', '--delete', ]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run_remote(cmd, host, project_dir, gene)
if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
run_remote(
'echo %s > %s' % (
gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']
), host, project_dir, gene
)
run_remote('sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene
)
)
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(
lambda x, y: x + y,
[timedelta(microseconds=int(r)) for r in times],
timedelta()
) / len(times) ** 2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print '==== Time results ===='
print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
print '''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)
if options.shutdown: # pragma: no cover
run_remote('sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue: # pragma: no cover
print "==== Till queue ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master'])
project_dir = gene.config['ec2']['code_folder']
run_remote(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene
)
if options.tiles_gen: # pragma: no cover
print "==== Generate tiles ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'slave'])
arguments.append("--daemonize")
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene)
)
if options.shutdown:
for p in processes:
p.communicate() # wait process end
else:
print 'Tile generation started in background'
if options.shutdown:
run_remote('sudo shutdown 0')
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'],
"""The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
{
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
},
"Tile generation controller")
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog='./buildout/bin/generate_amazon')
add_comon_options(parser)
parser.add_argument('--deploy-config',
default=None,
dest="deploy_config",
metavar="FILE",
help='path to the deploy configuration file')
parser.add_argument('--status',
default=False,
action="store_true",
help='display the SQS queue status and exit')
parser.add_argument('--disable-geodata',
default=True,
action="store_false",
dest="geodata",
help='disable geodata synchronisation')
parser.add_argument('--disable-code',
default=True,
action="store_false",
dest="deploy_code",
help='disable deploy application code')
parser.add_argument('--disable-database',
default=True,
action="store_false",
dest="deploy_database",
help='disable deploy database')
parser.add_argument('--disable-fillqueue',
default=True,
action="store_false",
dest="fill_queue",
help='disable queue filling')
parser.add_argument('--disable-tilesgen',
default=True,
action="store_false",
dest="tiles_gen",
help='disable tile generation')
parser.add_argument('--host',
default=None,
help='The host used to generate tiles')
parser.add_argument('--shutdown',
default=False,
action="store_true",
help='Shut done the remote host after the task.')
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print "EC2 not configured"
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if options.geodata and 'geodata_folder' in gene.config['ec2']:
print "==== Sync geodata ===="
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options, '-r',
gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code:
print "==== Sync and build code ===="
error = gene.validate(gene.config['ec2'],
'ec2',
'code_folder',
required=True)
if error:
exit(1) # pragma: no cover
cmd = [
'rsync',
'--delete',
]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run_remote(cmd, host, project_dir, gene)
if 'apache_content' in gene.config[
'ec2'] and 'apache_config' in gene.config['ec2']:
run_remote(
'echo %s > %s' % (gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']), host,
project_dir, gene)
run_remote('sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir,
gene))
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(lambda x, y: x + y,
[timedelta(microseconds=int(r))
for r in times], timedelta()) / len(times)**2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size],
0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print '==== Time results ===='
print 'A tile is generated in: %0.3f [ms]' % mean_time_ms
print 'Then mean generated tile size: %0.3f [kb]' % (mean_size_kb)
print '''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb)
if options.shutdown: # pragma: no cover
run_remote('sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue: # pragma: no cover
print "==== Till queue ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master'])
project_dir = gene.config['ec2']['code_folder']
run_remote(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir, gene)
if options.tiles_gen: # pragma: no cover
print "==== Generate tiles ===="
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'slave'])
arguments.append("--daemonize")
project_dir = gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
'./buildout/bin/generate_tiles ' +
' '.join([str(a) for a in arguments]), host, project_dir,
gene))
if options.shutdown:
for p in processes:
p.communicate() # wait process end
else:
print 'Tile generation started in background'
if options.shutdown:
run_remote('sudo shutdown 0')
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" % {
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
}, "Tile generation controller")
def main():
parser = ArgumentParser(
description='Used to generate the tiles from Amazon EC2, '
'and get the SQS queue status',
prog=sys.argv[0]
)
add_comon_options(parser)
parser.add_argument(
'--deploy-config', default=None, dest="deploy_config", metavar="FILE",
help='path to the deploy configuration file'
)
parser.add_argument(
'--status', default=False, action="store_true",
help='display the SQS queue status and exit'
)
parser.add_argument(
'--disable-geodata', default=True, action="store_false", dest="geodata",
help='disable geodata synchronisation'
)
parser.add_argument(
'--disable-code', default=True, action="store_false", dest="deploy_code",
help='disable deploy application code'
)
parser.add_argument(
'--disable-database', default=True, action="store_false", dest="deploy_database",
help='disable deploy database'
)
parser.add_argument(
'--disable-fillqueue', default=True, action="store_false", dest="fill_queue",
help='disable queue filling'
)
parser.add_argument(
'--disable-tilesgen', default=True, action="store_false", dest="tiles_gen",
help='disable tile generation'
)
parser.add_argument(
'--host', default=None,
help='The host used to generate tiles'
)
parser.add_argument(
'--shutdown', default=False, action="store_true",
help='Shut done the remote host after the task.'
)
parser.add_argument(
'--wait', default=False, action="store_true",
help='Wait that all the tasks will finish.'
)
parser.add_argument(
'--local', default=False, action="store_true",
help='Run the generation locally'
)
options = parser.parse_args()
gene = TileGeneration(options.config, options, layer_name=options.layer)
if options.status: # pragma: no cover
status(options, gene)
sys.exit(0)
if 'ec2' not in gene.config: # pragma: no cover
print("EC2 not configured")
sys.exit(1)
if options.deploy_config is None:
options.deploy_config = gene.config['ec2']['deploy_config']
if options.geodata:
options.geodata = not gene.config['ec2']['disable_geodata']
if options.deploy_code:
options.deploy_code = not gene.config['ec2']['disable_code']
if options.deploy_database:
options.deploy_database = not gene.config['ec2']['disable_database']
if options.fill_queue: # pragma: no cover
options.fill_queue = not gene.config['ec2']['disable_fillqueue']
if options.tiles_gen: # pragma: no cover
options.tiles_gen = not gene.config['ec2']['disable_tilesgen']
# start aws
if not options.host:
# TODO not implemented yet
host = aws_start(gene.config['ec2']['host_type']) # pragma: no cover
else:
host = options.host
if not options.local and options.geodata and 'geodata_folder' in gene.config['ec2']:
print("==== Sync geodata ====")
ssh_options = ''
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
ssh_options = gene.config['ec2']['ssh_options']
# sync geodata
run_local([
'rsync', '--delete', '-e', 'ssh ' + ssh_options,
'-r', gene.config['ec2']['geodata_folder'],
host + ':' + gene.config['ec2']['geodata_folder']
])
if options.deploy_code and not options.local:
print("==== Sync and build code ====")
error = gene.validate(gene.config['ec2'], 'ec2', 'code_folder', required=True)
if error:
exit(1) # pragma: no cover
cmd = ['rsync', '--delete', ]
if 'ssh_options' in gene.config['ec2']: # pragma: no cover
cmd += ['-e', 'ssh ' + gene.config['ec2']['ssh_options']]
ssh_options = gene.config['ec2']['ssh_options']
project_dir = gene.config['ec2']['code_folder']
cmd += ['-r', '.', host + ':' + project_dir]
run_local(cmd)
for cmd in gene.config['ec2']['build_cmds']:
run(options, cmd % environ, host, project_dir, gene)
if 'apache_content' in gene.config['ec2'] and 'apache_config' in gene.config['ec2']:
run(
options,
'echo %s > %s' % (
gene.config['ec2']['apache_content'],
gene.config['ec2']['apache_config']
), host, project_dir, gene
)
run(options, 'sudo apache2ctl graceful', host, project_dir, gene)
# deploy
if options.deploy_database and not options.local:
_deploy(gene, host)
if options.deploy_code or options.deploy_database \
or options.geodata and not options.local:
# TODO not implemented yet
create_snapshot(host, gene)
if options.time:
arguments = _get_arguments(options)
arguments.extend(['--role', 'local'])
arguments.extend(['--time', str(options.time)])
project_dir = None if options.local else gene.config['ec2']['code_folder']
processes = []
for i in range(gene.config['ec2']['number_process']):
processes.append(
run_remote_process(
"%sgenerate_tiles %s" % (
_get_path(),
' '.join([str(a) for a in arguments])
), host, project_dir, gene
)
)
tiles_size = []
times = []
for p in processes:
results = p.communicate()
if results[1] != '': # pragma: no cover
logger.debug('ERROR: %s' % results[1])
if PY3:
results = [r.decode('utf-8') for r in results]
results = (re.sub(u'\n[^\n]*\r', u'\n', results[0]), )
results = (re.sub(u'^[^\n]*\r', u'', results[0]), )
for r in results[0].split('\n'):
if r.startswith('time: '):
times.append(int(r.replace('time: ', '')))
elif r.startswith('size: '):
tiles_size.append(int(r.replace('size: ', '')))
if len(times) == 0: # pragma: no cover
logger.error("Not enough data")
sys.exit(1)
mean_time = reduce(
lambda x, y: x + y,
[timedelta(microseconds=int(r)) for r in times],
timedelta()
) / len(times) ** 2
mean_time_ms = mean_time.seconds * 1000 + mean_time.microseconds / 1000.0
mean_size = reduce(lambda x, y: x + y, [int(r) for r in tiles_size], 0) / len(tiles_size)
mean_size_kb = mean_size / 1024.0
print('==== Time results ====')
print('A tile is generated in: %0.3f [ms]' % mean_time_ms)
print('Then mean generated tile size: %0.3f [kb]' % (mean_size_kb))
print('''config:
cost:
tileonly_generation_time: %0.3f
tile_generation_time: %0.3f
metatile_generation_time: 0
tile_size: %0.3f''' % (mean_time_ms, mean_time_ms, mean_size_kb))
if options.shutdown: # pragma: no cover
run(options, 'sudo shutdown 0', host, project_dir, gene)
sys.exit(0)
if options.fill_queue and not options.local: # pragma: no cover
print("==== Till queue ====")
# TODO test
arguments = _get_arguments(options)
arguments.extend(['--role', 'master', '--quiet'])
project_dir = gene.config['ec2']['code_folder']
run_remote_process(
options,
"%sgenerate_tiles %s" % (
_get_path(),
' '.join([str(a) for a in arguments])
), host, project_dir, gene
)
sleep(5)
attributes = gene.get_sqs_queue().get_attributes()
print(
"\rTiles to generate: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
)
)
if options.tiles_gen: # pragma: no cover
print("==== Generate tiles ====")
if options.wait and not options.local:
print("")
class Status(Thread):
def run(self): # pragma: no cover
while True:
attributes = gene.get_sqs_queue().get_attributes()
print(
"\rTiles to generate/generating: %s/%s" % (
attributes['ApproximateNumberOfMessages'],
attributes['ApproximateNumberOfMessagesNotVisible'],
)
)
sleep(1)
status_thread = Status()
status_thread.setDaemon(True)
status_thread.start()
arguments = _get_arguments(options)
arguments.extend(['--quiet'])
if not options.local:
arguments.extend(['--role', 'slave'])
project_dir = None if options.local else gene.config['ec2']['code_folder']
threads = []
for i in range(gene.config['ec2']['number_process']):
if options.local:
threads.append(run_local_process(
"%sgenerate_tiles --local-process-number %i %s" % (
_get_path(),
i, ' '.join([str(a) for a in arguments])
)
))
else:
run_remote_process(
"%sgenerate_tiles %s" % (
_get_path(),
' '.join([str(a) for a in arguments])
), host, project_dir, gene
)
print('Tile generation started')
if options.shutdown:
run(options, 'sudo shutdown 0')
if options.wait and options.local:
while len(threads) > 0:
threads = [t for t in threads if t.is_alive()]
sleep(1)
if 'sns' in gene.config:
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'],
"""The tile generation is finish
Host: %(host)s
Command: %(cmd)s""" %
{
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv])
},
"Tile generation controller"
)
def generate_resume(self, layer):
if self._options.time is None:
if layer is not None:
all_dimensions = self._gene.get_all_dimensions(layer)
message = [
"The tile generation of layer '{}{}' is finish".format(
layer["name"],
"" if ((len(all_dimensions) == 1
and len(all_dimensions[0]) == 0)
or layer["type"] != "wms") else
" ({})".format(" - ".join([
", ".join(
["=".join(d) for d in dimensions.items()])
for dimensions in all_dimensions
])),
),
]
else:
message = ["The tile generation is finish"]
if self._options.role == "master": # pragma: no cover
message.append("Nb of generated jobs: {}".format(
self._count_tiles.nb))
elif layer.get(
"meta"
) if layer is not None else self._options.role == "slave":
message += [
"Nb generated metatiles: {}".format(
self._count_metatiles.nb),
"Nb metatiles dropped: {}".format(
self._count_metatiles_dropped.nb),
]
if self._options.role != "master":
message += [
"Nb generated tiles: {}".format(self._count_tiles.nb),
"Nb tiles dropped: {}".format(
self._count_tiles_dropped.nb),
]
if self._options.role in ("local", "slave"):
message += [
"Nb tiles stored: {}".format(
self._count_tiles_stored.nb),
"Nb tiles in error: {}".format(self._gene.error),
"Total time: {}".format(
duration_format(self._gene.duration)),
]
if self._count_tiles_stored.nb != 0:
message.append("Total size: {}".format(
size_format(self._count_tiles_stored.size)))
if self._count_tiles.nb != 0:
message.append("Time per tile: {:0.0f} ms".format(
(self._gene.duration / self._count_tiles.nb *
1000).seconds))
if self._count_tiles_stored.nb != 0:
message.append("Size per tile: {:0.0f} o".format(
self._count_tiles_stored.size /
self._count_tiles_stored.nb))
if not self._options.quiet and self._options.role in (
"local", "slave", "master") and message:
print("\n".join(message) + "\n")
if self._cache_tilestore is not None and hasattr(
self._cache_tilestore, "connection"):
self._cache_tilestore.connection.close()
if (self._options.role != "hash" and self._options.time is None
and "sns" in self._gene.config): # pragma: no cover
if "region" in self._gene.config["sns"]:
sns_client = boto3.client(
"sns", region_name=self._gene.config["sns"]["region"])
else:
sns_client = boto3.client("sns")
sns_message = [message[0]]
sns_message += [
"Layer: {}".format(
layer["name"] if layer is not None else "(All layers)"),
"Role: {}".format(self._options.role),
"Host: {}".format(socket.getfqdn()),
"Command: {}".format(" ".join([quote(arg)
for arg in sys.argv])),
]
sns_message += message[1:]
sns_client.publish(
TopicArn=self._gene.config["sns"]["topic"],
Message="\n".join(sns_message),
Subject="Tile generation ({layer} - {role})".format(
role=self._options.role,
layer=layer["name"]
if layer is not None else "All layers"),
)
def generate_resume(self, layer):
if self._options.time is None:
if layer is not None:
all_dimensions = self._gene.get_all_dimensions(layer)
message = [
"The tile generation of layer '{}{}' is finish".format(
layer['name'],
"" if (
(len(all_dimensions) == 1 and len(all_dimensions[0]) == 0)
or layer['type'] != 'wms'
)
else " ({})".format(
" - ".join([
", ".join(
["=".join(d) for d in dimensions.items()]
)
for dimensions in all_dimensions
])
)
),
]
else:
message = [
"The tile generation is finish"
]
if self._options.role == "master": # pragma: no cover
message.append("Nb of generated jobs: {}".format(self._count_tiles.nb))
elif layer.get('meta') if layer is not None else self._options.role == "slave":
message += [
"Nb generated metatiles: {}".format(self._count_metatiles.nb),
"Nb metatiles dropped: {}".format(self._count_metatiles_dropped.nb),
]
if self._options.role != "master":
message += [
"Nb generated tiles: {}".format(self._count_tiles.nb),
"Nb tiles dropped: {}".format(self._count_tiles_dropped.nb),
]
if self._options.role in ('local', 'slave'):
message += [
"Nb tiles stored: {}".format(self._count_tiles_stored.nb),
"Nb tiles in error: {}".format(self._gene.error),
"Total time: {}".format(duration_format(self._gene.duration)),
]
if self._count_tiles_stored.nb != 0:
message.append("Total size: {}".format(size_format(self._count_tiles_stored.size)))
if self._count_tiles.nb != 0:
message.append("Time per tile: {:0.0f} ms".format(
(self._gene.duration / self._count_tiles.nb * 1000).seconds)
)
if self._count_tiles_stored.nb != 0:
message.append("Size per tile: {:0.0f} o".format(
self._count_tiles_stored.size / self._count_tiles_stored.nb)
)
if not self._options.quiet and self._options.role in ('local', 'slave', 'master') and message:
print("\n".join(message) + "\n")
if self._cache_tilestore is not None and hasattr(self._cache_tilestore, 'connection'):
self._cache_tilestore.connection.close()
if self._options.role != 'hash' and \
self._options.time is None and \
'sns' in self._gene.config: # pragma: no cover
if 'region' in self._gene.config['sns']:
sns_client = boto3.client('sns', region_name=self._gene.config['sns']['region'])
else:
sns_client = boto3.client('sns')
sns_message = [message[0]]
sns_message += [
"Layer: {}".format(layer['name'] if layer is not None else "(All layers)"),
"Role: {}".format(self._options.role),
"Host: {}".format(socket.getfqdn()),
"Command: {}".format(' '.join([quote(arg) for arg in sys.argv])),
]
sns_message += message[1:]
sns_client.publish(TopicArn=self._gene.config['sns']['topic'], Message="\n".join(sns_message),
Subject="Tile generation ({layer} - {role})".format(
role=self._options.role,
layer=layer['name'] if layer is not None else "All layers"))
def gene(self, options, gene, layer):
count_metatiles = None
count_metatiles_dropped = Count()
count_tiles = None
count_tiles_dropped = Count()
if options.role == 'slave' or options.get_hash or options.get_bbox:
gene.layer = gene.layers[layer]
else:
gene.set_layer(layer, options)
if options.get_bbox:
try:
tilecoord = parse_tilecoord(options.get_bbox)
print \
"Tile bounds: [%i,%i,%i,%i]" % \
gene.layer['grid_ref']['obj'].extent(tilecoord)
exit()
except ValueError as e: # pragma: no cover
exit(
"Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r"
% (options.get_bbox, e))
if options.get_hash:
options.role = 'hash'
options.test = 1
sqs_tilestore = None
if options.role in ('master', 'slave'):
# Create SQS queue
sqs_tilestore = SQSTileStore(
gene.get_sqs_queue()) # pragma: no cover
cache_tilestore = None
if options.role in ('local', 'slave'):
cache_tilestore = gene.get_tilesstore(options.cache)
meta = gene.layer['meta']
if options.tiles:
gene.set_store(TilesFileStore(options.tiles))
elif options.role in ('local', 'master'):
# Generate a stream of metatiles
gene.init_tilecoords()
gene.add_geom_filter()
elif options.role == 'slave':
# Get the metatiles from the SQS queue
gene.set_store(sqs_tilestore) # pragma: no cover
elif options.role == 'hash':
try:
z, x, y = (int(v) for v in options.get_hash.split('/'))
if meta:
gene.set_tilecoords(
[TileCoord(z, x, y, gene.layer['meta_size'])])
else:
gene.set_tilecoords([TileCoord(z, x, y)])
except ValueError as e: # pragma: no cover
exit("Tile '%s' is not in the format 'z/x/y'\n%r" %
(options.get_hash, e))
# At this stage, the tilestream contains metatiles that intersect geometry
gene.add_logger()
count_metatiles = gene.counter()
if options.role == 'master': # pragma: no cover
# Put the metatiles into the SQS queue
gene.put(sqs_tilestore)
elif options.role in ('local', 'slave', 'hash'):
if gene.layer['type'] == 'wms':
params = gene.layer['params'].copy()
if 'STYLES' not in params:
params['STYLES'] = ','.join(gene.layer['wmts_style']
for l in gene.layer['layers'])
if gene.layer['generate_salt']:
params['SALT'] = str(random.randint(0, sys.maxint))
for dim in gene.layer['dimensions']:
params[dim['name']] = dim['value']
for dim in gene.options.dimensions:
dim = dim.split('=')
if len(dim) != 2: # pragma: no cover
exit('the DIMENTIONS option should be like this '
'DATE=2013 VERSION=13.')
params[dim[0]] = dim[1]
# Get the metatile image from the WMS server
gene.get(
URLTileStore(
tilelayouts=(WMSTileLayout(
url=gene.layer['url'],
layers=','.join(gene.layer['layers']),
srs=gene.layer['grid_ref']['srs'],
format=gene.layer['mime_type'],
border=gene.layer['meta_buffer'] if meta else 0,
tilegrid=gene.get_grid()['obj'],
params=params,
), ),
headers=gene.layer['headers'],
), "Get tile from WMS")
elif gene.layer['type'] == 'mapnik':
from tilecloud.store.mapnik_ import MapnikTileStore
from tilecloud_chain.mapnik_ import MapnikDropActionTileStore
grid = gene.get_grid()
if gene.layer['output_format'] == 'grid':
gene.get(
MapnikDropActionTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer']
if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
resolution=gene.layer['resolution'],
layers_fields=gene.layer['layers_fields'],
drop_empty_utfgrid=gene.
layer['drop_empty_utfgrid'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
proj4_literal=grid['proj4_literal'],
), "Create Mapnik grid tile")
else:
gene.get(
MapnikTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer']
if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
proj4_literal=grid['proj4_literal'],
), "Create Mapnik tile")
def wrong_content_type_to_error(tile):
if tile is not None and tile.content_type is not None \
and tile.content_type.find("image/") != 0:
if tile.content_type.find(
"application/vnd.ogc.se_xml") == 0:
tile.error = "WMS server error: %s" % (
self._re_rm_xml_tag.sub('', tile.data))
else: # pragma: no cover
tile.error = "%s is not an image format, error: %s" % (
tile.content_type, tile.data)
return tile
gene.imap(wrong_content_type_to_error)
# Handle errors
gene.add_error_filters()
if meta:
if options.role == 'hash':
gene.imap(HashLogger('empty_metatile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_metatile_detection' in gene.layer:
empty_tile = gene.layer['empty_metatile_detection']
gene.imap(
HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_metatiles_dropped,
))
def add_elapsed_togenerate(metatile):
if metatile is not None:
metatile.elapsed_togenerate = metatile.tilecoord.n**2
return True
return False # pragma: no cover
gene.ifilter(add_elapsed_togenerate)
# Split the metatile image into individual tiles
gene.add_metatile_splitter()
gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))
# Handle errors
gene.add_error_filters()
self.count_tiles = gene.counter()
if 'pre_hash_post_process' in gene.layer:
gene.process(gene.layer['pre_hash_post_process'])
if options.role == 'hash':
gene.imap(HashLogger('empty_tile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_tile_detection' in gene.layer:
empty_tile = gene.layer['empty_tile_detection']
gene.imap(
HashDropper(
empty_tile['size'],
empty_tile['hash'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
))
gene.process()
if options.role in ('local', 'slave'):
gene.add_error_filters()
gene.ifilter(DropEmpty(gene))
count_tiles = gene.counter(size=True)
if options.time:
def log_size(tile):
sys.stdout.write('size: %i\n' % len(tile.data))
return tile
gene.imap(log_size)
gene.put(cache_tilestore, "Store the tile")
else:
count_tiles = gene.counter(size=True)
gene.add_error_filters()
if options.generated_tiles_file: # pragma: no cover
generated_tiles_file = open(options.generated_tiles_file, 'a')
def do(tile):
generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
return tile
gene.imap(do)
if options.role == 'slave': # pragma: no cover
if meta:
def decr_tile_in_metatile(tile):
tile.metatile.elapsed_togenerate -= 1
if tile.metatile.elapsed_togenerate == 0:
sqs_tilestore.delete_one(tile.metatile)
return True
gene.ifilter(decr_tile_in_metatile)
else:
gene.delete(sqs_tilestore)
if options.time is not None:
class LogTime:
n = 0
t1 = None
def __call__(self, tile):
self.n += 1
if self.n == options.time:
self.t1 = datetime.now()
elif self.n == 2 * options.time:
t2 = datetime.now()
d = (t2 - self.t1) / options.time
sys.stdout.write(
'time: %i\n' %
((d.days * 24 * 3600 + d.seconds) * 1000000 +
d.microseconds))
return tile
gene.imap(LogTime())
gene.consume(options.time * 3)
else:
gene.consume()
if not options.quiet and options.role in ('local', 'slave'):
nb_tiles = count_tiles.nb + count_tiles_dropped.nb
print """The tile generation of layer '%s' is finish
%sNb generated tiles: %i
Nb tiles dropped: %i
Nb tiles stored: %i
Nb error: %i
Total time: %s
Total size: %s
Time per tiles: %i ms
Size per tile: %i o
""" % \
(
gene.layer['name'],
"""Nb generated metatiles: %i
Nb metatiles dropped: %i
""" %
(
count_metatiles.nb, count_metatiles_dropped.nb
) if meta else '',
nb_tiles,
count_tiles_dropped.nb,
count_tiles.nb,
gene.error,
duration_format(gene.duration),
size_format(count_tiles.size),
(gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0,
count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1
)
if cache_tilestore is not None and hasattr(cache_tilestore,
'connection'):
cache_tilestore.connection.close()
if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(
gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'], """The tile generation is finish
Layer: %(layer)s
Role: %(role)s
Host: %(host)s
Command: %(cmd)s
%(meta)sNb generated tiles: %(nb_tiles)i
Nb tiles dropped: %(nb_tiles_dropped)i
Total time: %(duration)s [s]
Time per tiles: %(tile_duration)i [ms]""" % {
'role':
options.role,
'layer':
gene.layer['name'],
'host':
socket.getfqdn(),
'cmd':
' '.join([quote(arg) for arg in sys.argv]),
'meta':
"""Nb generated metatiles: %(nb_metatiles)i
Nb metatiles dropped: %(nb_metatiles_dropped)i
""" % {
'nb_metatiles': count_metatiles.nb,
'nb_metatiles_dropped': count_metatiles_dropped.nb,
} if meta else '',
'nb_tiles':
nb_tiles if meta else count_metatiles.nb,
'nb_tiles_dropped':
count_tiles_dropped.nb
if meta else count_metatiles_dropped.nb,
'duration':
duration_format(gene.duration),
'tile_duration': (gene.duration / nb_tiles *
1000).seconds if nb_tiles != 0 else 0,
}, "Tile generation (%(layer)s - %(role)s)" % {
'role': options.role,
'layer': gene.layer['name']
})
def _gene(self, options, gene, layer, dimensions={}):
count_metatiles = None
count_metatiles_dropped = Count()
count_tiles = None
count_tiles_dropped = Count()
count_tiles_stored = None
if options.get_bbox:
try:
tilecoord = parse_tilecoord(options.get_bbox)
print(
"Tile bounds: [%i,%i,%i,%i]" %
gene.layer['grid_ref']['obj'].extent(tilecoord)
)
exit()
except ValueError as e: # pragma: no cover
exit(
"Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" %
(options.get_bbox, e)
)
if options.get_hash:
options.role = 'hash'
options.test = 1
sqs_tilestore = None
if options.role in ('master', 'slave'):
# Create SQS queue
sqs_tilestore = SQSTileStore(gene.get_sqs_queue()) # pragma: no cover
cache_tilestore = None
if options.role in ('local', 'slave'):
cache_tilestore = gene.get_tilesstore(options.cache, dimensions)
meta = gene.layer['meta']
if options.tiles:
gene.set_store(TilesFileStore(options.tiles))
elif options.role in ('local', 'master'):
# Generate a stream of metatiles
gene.init_tilecoords()
gene.add_geom_filter()
if options.local_process_number is not None: # pragma: no cover
gene.add_local_process_filter()
elif options.role == 'slave':
# Get the metatiles from the SQS queue
gene.set_store(sqs_tilestore) # pragma: no cover
elif options.role == 'hash':
try:
z, x, y = (int(v) for v in options.get_hash.split('/'))
if meta:
gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])])
else:
gene.set_tilecoords([TileCoord(z, x, y)])
except ValueError as e: # pragma: no cover
exit(
"Tile '%s' is not in the format 'z/x/y'\n%r" %
(options.get_hash, e)
)
# At this stage, the tilestream contains metatiles that intersect geometry
gene.add_logger()
count_metatiles = gene.counter()
if options.role == 'master': # pragma: no cover
# Put the metatiles into the SQS queue
gene.put(sqs_tilestore)
count_tiles = gene.counter()
elif options.role in ('local', 'slave', 'hash'):
if gene.layer['type'] == 'wms':
params = gene.layer['params'].copy()
if 'STYLES' not in params:
params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers'].split(','))
if gene.layer['generate_salt']:
params['SALT'] = str(random.randint(0, 999999))
params.update(dimensions)
# Get the metatile image from the WMS server
gene.get(URLTileStore(
tilelayouts=(WMSTileLayout(
url=gene.layer['url'],
layers=gene.layer['layers'],
srs=gene.layer['grid_ref']['srs'],
format=gene.layer['mime_type'],
border=gene.layer['meta_buffer'] if meta else 0,
tilegrid=gene.get_grid()['obj'],
params=params,
),),
headers=gene.layer['headers'],
), "Get tile from WMS")
elif gene.layer['type'] == 'mapnik': # pragma: no cover
from tilecloud.store.mapnik_ import MapnikTileStore
from tilecloud_chain.mapnik_ import MapnikDropActionTileStore
grid = gene.get_grid()
if gene.layer['output_format'] == 'grid':
count_tiles = gene.counter()
gene.get(MapnikDropActionTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer'] if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
resolution=gene.layer['resolution'],
layers_fields=gene.layer['layers_fields'],
drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
proj4_literal=grid['proj4_literal'],
), "Create Mapnik grid tile")
else:
gene.get(MapnikTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer'] if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
proj4_literal=grid['proj4_literal'],
), "Create Mapnik tile")
def wrong_content_type_to_error(tile):
if tile is not None and tile.content_type is not None \
and tile.content_type.find("image/") != 0:
if tile.content_type.find("application/vnd.ogc.se_xml") == 0:
tile.error = "WMS server error: %s" % (
self._re_rm_xml_tag.sub(
'', tile.data.decode('utf-8') if PY3 else tile.data
)
)
else: # pragma: no cover
tile.error = "%s is not an image format, error: %s" % (
tile.content_type,
tile.data
)
return tile
gene.imap(wrong_content_type_to_error)
# Handle errors
gene.add_error_filters()
if meta:
if options.role == 'hash':
gene.imap(HashLogger('empty_metatile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_metatile_detection' in gene.layer:
empty_tile = gene.layer['empty_metatile_detection']
gene.imap(HashDropper(
empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_metatiles_dropped,
))
def add_elapsed_togenerate(metatile):
if metatile is not None:
metatile.elapsed_togenerate = metatile.tilecoord.n ** 2
return True
return False # pragma: no cover
gene.ifilter(add_elapsed_togenerate)
# Split the metatile image into individual tiles
gene.add_metatile_splitter()
gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))
# Handle errors
gene.add_error_filters()
if gene.layer['type'] != 'mapnik' or gene.layer['output_format'] != 'grid':
count_tiles = gene.counter()
if 'pre_hash_post_process' in gene.layer: # pragma: no cover
gene.process(gene.layer['pre_hash_post_process'])
if options.role == 'hash':
gene.imap(HashLogger('empty_tile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_tile_detection' in gene.layer:
empty_tile = gene.layer['empty_tile_detection']
gene.imap(HashDropper(
empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
))
gene.process()
else: # pragma: no cover
count_tiles = gene.counter()
if options.role in ('local', 'slave'):
gene.add_error_filters()
gene.ifilter(DropEmpty(gene))
count_tiles_stored = gene.counter(size=True)
if options.time:
def log_size(tile):
sys.stdout.write('size: %i\n' % len(tile.data))
return tile
gene.imap(log_size)
gene.put(cache_tilestore, "Store the tile")
gene.add_error_filters()
if options.generated_tiles_file: # pragma: no cover
generated_tiles_file = open(options.generated_tiles_file, 'a')
def do(tile):
generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
return tile
gene.imap(do)
if options.role == 'slave': # pragma: no cover
if meta:
def decr_tile_in_metatile(tile):
tile.metatile.elapsed_togenerate -= 1
if tile.metatile.elapsed_togenerate == 0:
sqs_tilestore.delete_one(tile.metatile)
return True
gene.ifilter(decr_tile_in_metatile)
else:
gene.delete(sqs_tilestore)
message = []
if options.time is not None:
class LogTime:
n = 0
t1 = None
def __call__(self, tile):
self.n += 1
if self.n == options.time:
self.t1 = datetime.now()
elif self.n == 2 * options.time:
t2 = datetime.now()
d = (t2 - self.t1) / options.time
sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds))
return tile
gene.imap(LogTime())
gene.consume(options.time * 3)
else:
gene.consume()
message = [
"The tile generation of layer '{}{}' is finish".format(
gene.layer['name'],
"" if len(dimensions) == 0 or gene.layer['type'] != 'wms'
else " (%s)" % ", ".join(["=".join(d) for d in dimensions.items()])
),
]
if options.role == "master": # pragma: no cover
message.append("Nb of generated jobs: {}".format(count_tiles.nb))
else:
if meta:
message += [
"Nb generated metatiles: {}".format(count_metatiles.nb),
"Nb metatiles dropped: {}".format(count_metatiles_dropped.nb),
]
message += [
"Nb generated tiles: {}".format(count_tiles.nb),
"Nb tiles dropped: {}".format(count_tiles_dropped.nb),
]
if options.role in ('local', 'slave'):
message += [
"Nb tiles stored: {}".format(count_tiles_stored.nb),
"Nb tiles in error: {}".format(gene.error),
"Total time: {}".format(duration_format(gene.duration)),
]
if count_tiles_stored.nb != 0:
message.append("Total size: {}".format(size_format(count_tiles_stored.size)))
if count_tiles.nb != 0:
message.append("Time per tile: {:0.0f} ms".format(
(gene.duration / count_tiles.nb * 1000).seconds)
)
if count_tiles_stored.nb != 0:
message.append("Size per tile: {:0.0f} o".format(
count_tiles_stored.size / count_tiles_stored.nb)
)
if not options.quiet and options.role in ('local', 'slave'):
print("\n".join(message) + "\n")
if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'):
cache_tilestore.connection.close()
if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(gene.config['sns']['region'])
else:
connection = boto.connect_sns()
sns_message = [message[0]]
sns_message += [
"Layer: {}".format(gene.layer['name']),
"Role: {}".format(options.role),
"Host: {}".format(socket.getfqdn()),
"Command: {}".format(' '.join([quote(arg) for arg in sys.argv])),
]
sns_message += message[1:]
connection.publish(
gene.config['sns']['topic'],
"\n".join(sns_message),
"Tile generation (%(layer)s - %(role)s)" % {
'role': options.role,
'layer': gene.layer['name']
}
)
def gene(self, options, gene, layer):
count_metatiles = None
count_metatiles_dropped = Count()
count_tiles = None
count_tiles_dropped = Count()
if options.role == 'slave' or options.get_hash or options.get_bbox:
gene.layer = gene.layers[layer]
else:
gene.set_layer(layer, options)
if options.get_bbox:
try:
tilecoord = parse_tilecoord(options.get_bbox)
print(
"Tile bounds: [%i,%i,%i,%i]" %
gene.layer['grid_ref']['obj'].extent(tilecoord)
)
exit()
except ValueError as e: # pragma: no cover
exit(
"Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n\n%r" %
(options.get_bbox, e)
)
if options.get_hash:
options.role = 'hash'
options.test = 1
sqs_tilestore = None
if options.role in ('master', 'slave'):
# Create SQS queue
sqs_tilestore = SQSTileStore(gene.get_sqs_queue()) # pragma: no cover
cache_tilestore = None
if options.role in ('local', 'slave'):
cache_tilestore = gene.get_tilesstore(options.cache)
meta = gene.layer['meta']
if options.tiles:
gene.set_store(TilesFileStore(options.tiles))
elif options.role in ('local', 'master'):
# Generate a stream of metatiles
gene.init_tilecoords()
gene.add_geom_filter()
if options.local_process_number is not None: # pragma: no cover
gene.add_local_process_filter()
elif options.role == 'slave':
# Get the metatiles from the SQS queue
gene.set_store(sqs_tilestore) # pragma: no cover
elif options.role == 'hash':
try:
z, x, y = (int(v) for v in options.get_hash.split('/'))
if meta:
gene.set_tilecoords([TileCoord(z, x, y, gene.layer['meta_size'])])
else:
gene.set_tilecoords([TileCoord(z, x, y)])
except ValueError as e: # pragma: no cover
exit(
"Tile '%s' is not in the format 'z/x/y'\n%r" %
(options.get_hash, e)
)
# At this stage, the tilestream contains metatiles that intersect geometry
gene.add_logger()
count_metatiles = gene.counter()
if options.role == 'master': # pragma: no cover
# Put the metatiles into the SQS queue
gene.put(sqs_tilestore)
elif options.role in ('local', 'slave', 'hash'):
if gene.layer['type'] == 'wms':
params = gene.layer['params'].copy()
if 'STYLES' not in params:
params['STYLES'] = ','.join(gene.layer['wmts_style'] for l in gene.layer['layers'])
if gene.layer['generate_salt']:
params['SALT'] = str(random.randint(0, sys.maxint))
for dim in gene.layer['dimensions']:
params[dim['name']] = dim['value']
for dim in gene.options.dimensions:
dim = dim.split('=')
if len(dim) != 2: # pragma: no cover
exit(
'the DIMENTIONS option should be like this '
'DATE=2013 VERSION=13.'
)
params[dim[0]] = dim[1]
# Get the metatile image from the WMS server
gene.get(URLTileStore(
tilelayouts=(WMSTileLayout(
url=gene.layer['url'],
layers=','.join(gene.layer['layers']),
srs=gene.layer['grid_ref']['srs'],
format=gene.layer['mime_type'],
border=gene.layer['meta_buffer'] if meta else 0,
tilegrid=gene.get_grid()['obj'],
params=params,
),),
headers=gene.layer['headers'],
), "Get tile from WMS")
elif gene.layer['type'] == 'mapnik':
from tilecloud.store.mapnik_ import MapnikTileStore
from tilecloud_chain.mapnik_ import MapnikDropActionTileStore
grid = gene.get_grid()
if gene.layer['output_format'] == 'grid':
gene.get(MapnikDropActionTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer'] if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
resolution=gene.layer['resolution'],
layers_fields=gene.layer['layers_fields'],
drop_empty_utfgrid=gene.layer['drop_empty_utfgrid'],
store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
proj4_literal=grid['proj4_literal'],
), "Create Mapnik grid tile")
else:
gene.get(MapnikTileStore(
tilegrid=grid['obj'],
mapfile=gene.layer['mapfile'],
image_buffer=gene.layer['meta_buffer'] if meta else 0,
data_buffer=gene.layer['data_buffer'],
output_format=gene.layer['output_format'],
proj4_literal=grid['proj4_literal'],
), "Create Mapnik tile")
def wrong_content_type_to_error(tile):
if tile is not None and tile.content_type is not None \
and tile.content_type.find("image/") != 0:
if tile.content_type.find("application/vnd.ogc.se_xml") == 0:
tile.error = "WMS server error: %s" % (
self._re_rm_xml_tag.sub('', tile.data)
)
else: # pragma: no cover
tile.error = "%s is not an image format, error: %s" % (
tile.content_type,
tile.data
)
return tile
gene.imap(wrong_content_type_to_error)
# Handle errors
gene.add_error_filters()
if meta:
if options.role == 'hash':
gene.imap(HashLogger('empty_metatile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_metatile_detection' in gene.layer:
empty_tile = gene.layer['empty_metatile_detection']
gene.imap(HashDropper(
empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_metatiles_dropped,
))
def add_elapsed_togenerate(metatile):
if metatile is not None:
metatile.elapsed_togenerate = metatile.tilecoord.n ** 2
return True
return False # pragma: no cover
gene.ifilter(add_elapsed_togenerate)
# Split the metatile image into individual tiles
gene.add_metatile_splitter()
gene.imap(Logger(logger, logging.INFO, '%(tilecoord)s'))
# Handle errors
gene.add_error_filters()
self.count_tiles = gene.counter()
if 'pre_hash_post_process' in gene.layer:
gene.process(gene.layer['pre_hash_post_process'])
if options.role == 'hash':
gene.imap(HashLogger('empty_tile_detection'))
elif not options.near:
# Discard tiles with certain content
if 'empty_tile_detection' in gene.layer:
empty_tile = gene.layer['empty_tile_detection']
gene.imap(HashDropper(
empty_tile['size'], empty_tile['hash'], store=cache_tilestore,
queue_store=sqs_tilestore,
count=count_tiles_dropped,
))
gene.process()
if options.role in ('local', 'slave'):
gene.add_error_filters()
gene.ifilter(DropEmpty(gene))
count_tiles = gene.counter(size=True)
if options.time:
def log_size(tile):
sys.stdout.write('size: %i\n' % len(tile.data))
return tile
gene.imap(log_size)
gene.put(cache_tilestore, "Store the tile")
else:
count_tiles = gene.counter(size=True)
gene.add_error_filters()
if options.generated_tiles_file: # pragma: no cover
generated_tiles_file = open(options.generated_tiles_file, 'a')
def do(tile):
generated_tiles_file.write('%s\n' % (tile.tilecoord, ))
return tile
gene.imap(do)
if options.role == 'slave': # pragma: no cover
if meta:
def decr_tile_in_metatile(tile):
tile.metatile.elapsed_togenerate -= 1
if tile.metatile.elapsed_togenerate == 0:
sqs_tilestore.delete_one(tile.metatile)
return True
gene.ifilter(decr_tile_in_metatile)
else:
gene.delete(sqs_tilestore)
if options.time is not None:
class LogTime:
n = 0
t1 = None
def __call__(self, tile):
self.n += 1
if self.n == options.time:
self.t1 = datetime.now()
elif self.n == 2 * options.time:
t2 = datetime.now()
d = (t2 - self.t1) / options.time
sys.stdout.write('time: %i\n' % ((d.days * 24 * 3600 + d.seconds) * 1000000 + d.microseconds))
return tile
gene.imap(LogTime())
gene.consume(options.time * 3)
else:
gene.consume()
if not options.quiet and options.role in ('local', 'slave'):
nb_tiles = count_tiles.nb + count_tiles_dropped.nb
print(
"""The tile generation of layer '%s' is finish
%sNb generated tiles: %i
Nb tiles dropped: %i
Nb tiles stored: %i
Nb error: %i
Total time: %s
Total size: %s
Time per tiles: %i ms
Size per tile: %i o
""" % (
gene.layer['name'],
"""Nb generated metatiles: %i
Nb metatiles dropped: %i
""" %
(
count_metatiles.nb, count_metatiles_dropped.nb
) if meta else '',
nb_tiles,
count_tiles_dropped.nb,
count_tiles.nb,
gene.error,
duration_format(gene.duration),
size_format(count_tiles.size),
(gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0,
count_tiles.size / count_tiles.nb if count_tiles.nb != 0 else -1
)
)
if cache_tilestore is not None and hasattr(cache_tilestore, 'connection'):
cache_tilestore.connection.close()
if options.role != 'hash' and options.time is None and 'sns' in gene.config: # pragma: no cover
if 'region' in gene.config['sns']:
connection = sns.connect_to_region(gene.config['sns']['region'])
else:
connection = boto.connect_sns()
connection.publish(
gene.config['sns']['topic'],
"""The tile generation is finish
Layer: %(layer)s
Role: %(role)s
Host: %(host)s
Command: %(cmd)s
%(meta)sNb generated tiles: %(nb_tiles)i
Nb tiles dropped: %(nb_tiles_dropped)i
Total time: %(duration)s [s]
Time per tiles: %(tile_duration)i [ms]""" %
{
'role': options.role,
'layer': gene.layer['name'],
'host': socket.getfqdn(),
'cmd': ' '.join([quote(arg) for arg in sys.argv]),
'meta': """Nb generated metatiles: %(nb_metatiles)i
Nb metatiles dropped: %(nb_metatiles_dropped)i
""" %
{
'nb_metatiles': count_metatiles.nb,
'nb_metatiles_dropped': count_metatiles_dropped.nb,
} if meta else '',
'nb_tiles': nb_tiles if meta else count_metatiles.nb,
'nb_tiles_dropped': count_tiles_dropped.nb if meta else count_metatiles_dropped.nb,
'duration': duration_format(gene.duration),
'tile_duration': (gene.duration / nb_tiles * 1000).seconds if nb_tiles != 0 else 0,
},
"Tile generation (%(layer)s - %(role)s)" % {
'role': options.role,
'layer': gene.layer['name']
}
)
