def _create(self, config, style_name, storage, formats, format_args):
renderer = None
rendering_system = config.get(style_name, "system")
if rendering_system == 'mapnik':
default_style = config.get(style_name, "default_style")
mask_style = config.get(style_name, "mask_style") if config.has_option(style_name, "mask_style") else None
renderer = mapnik.Renderer(default_style, mask_style)
for (polygon, style) in config.items(style_name):
# teh uglies. to be refactored. and not only because the keys are downcased...
if polygon != "default_style" and polygon != "mask_style" and polygon != "system":
renderer.add_region(polygon, style, polygon)
elif rendering_system == 'terrain':
renderer = terrain.Renderer(config.get(style_name, "tr_config"))
elif rendering_system == 'aerial':
renderer = aerial.Renderer(config.get(style_name, "ae_config"))
elif rendering_system == 'composite':
layers = config.get(style_name, 'layers')
layers = csv.reader([layers], skipinitialspace=True).next()
background = config.get(style_name, 'background') if 'background' in dict(config.items(style_name)) else None
renderer = composite.Renderer(layers, self, background)
elif rendering_system == 'coverages':
coverage_conf = dict(config.items('coverages'))
vendor_conf = dict(config.items(style_name))
del vendor_conf['system']
renderer = coverages.Renderer(coverage_conf, vendor_conf, self)
else:
mq_logging.error("'%s' is not a known rendering system." % rendering_system)
return renderer
def process(self, tile):
z = str(tile.z)
x_range = range( tile.x, tile.x + tile.dimensions[0] )
y_range = range( tile.y, tile.y + tile.dimensions[1] )
urls = [ [self.url % {'z':z, 'x':str(x), 'y':str(y)}, x - tile.x, y - tile.y] for x in x_range for y in y_range ]
try:
subTiles = map(getTile, urls)
except Exception as detail:
mq_logging.error('Could not read tile from aerial source: %s' % str(detail))
raise
if len( subTiles ) < tile.dimensions[0] * tile.dimensions[1] :
mq_logging.warning("Result length too short, aborting")
return None, None
#
# make a composite image
#
features = None
image = PIL.Image.new( 'RGBA', tile.size )
for infoImage in subTiles:
imageSubTile = PIL.Image.open( StringIO.StringIO( infoImage[0] ) )
image.paste( imageSubTile, ( infoImage[1] * TILE_SIZE, infoImage[2] * TILE_SIZE ) )
return RenderResult.from_image(tile, image)
def notify (job, queue):
notified = False
while notified == False:
try:
queue.notify(job)
notified = True
except RuntimeError, e:
error_message = e.message
if error_message.lower().find ("deadlock") == -1:
raise
else:
mq_logging.error("deadlock error notify: %s" % error_message)
def add_region(self,name,mapfile,wkt_mask_file):
# ignore attempts to add regions when no mask has been
# specified with a warning.
if self.mask_map is not None:
try:
m = Map(0,0)
load_map(m, mapfile)
mask = Mask(wkt_mask_file)
self.regions.append((name,m,mask))
except Exception as ex:
mq_logging.error("Exception caught adding region (%s)" % str(ex))
else:
mq_logging.warning("Cannot add mask (%s) because no mask_style was configured." % name)
def __init__(self, config, formats, format_args, storage):
self.renderers = {}
# the config file gives the names of all the styles, separated by
# commas, in the [worker] section. each of these styles is then
# expected to be a section name, naming the style. this is what
# the job is matched against when it comes in. one style name is
# one renderer.
style_names = csv.reader([config.get('worker', 'styles')],skipinitialspace=True).next()
# the styles which will be saved to storage when they are done.
# note that it's possible for stuff to not be saved to storage
# and this might be reasonable when the work involved in making
# that tile is very low.
save_names = csv.reader([config.get('worker', 'saved_styles')],skipinitialspace=True).next()
# read only styles
try:
read_only_names = csv.reader([config.get('worker', 'read_only_styles')],skipinitialspace=True).next()
except:
read_only_names = []
for style_name in style_names:
mq_logging.info("Loading '%s' style" % style_name)
try :
# sanity check - there's no real problem with styles being
# defined twice, but it probably indicates a typo or other
# error in the config file.
if style_name in self.renderers:
raise Exception("style '%s' is defined twice." % style_name)
# if the style is only to be read from storage
if style_name in read_only_names:
renderer = StorageRenderer(storage, formats[style_name], format_args, None)
else:
renderer = self._create(config, style_name, storage, formats, format_args)
# if the style name is in the formats config then it's supposed
# to be saved to storage. so wrap the renderer in a saving
# renderer.
if (renderer is not None) and (style_name in save_names):
renderer = StorageRenderer(storage, formats[style_name], format_args, renderer)
if renderer is None:
raise Exception("unable to create renderer from style '%s'." % style_name)
self.renderers[style_name] = renderer
except Exception as ex:
mq_logging.error("failed to load worker configuration for '%s' from: %s (%s)" % (style_name,config, str(ex)))
sys.exit(1)
def save_meta(storage, job, tiles, metaData, formats, size):
meta = make_meta(job, tiles, metaData, formats, size)
# Send the meta tile to storage
success = storage.put_meta(job, meta)
# Make sure we know about it if the storage doesn't work. The cluster
# will continue working, as the data can be sent back via the broker,
# but it's helpful to have something in the logs so we know what's
# going on...
if not success:
mq_logging.error("Failed to save meta tile to storage (%d:%d:%d:%s tile-size=%d)" % \
(job.z,job.x,job.y,job.style,len(job.data)))
# Return the meta tile
return meta
def __init__( self, configFileName ):
self.host = "mq-aerial-lm01.ihost.aol.com"
self.port = 5005
try:
configSettings = ConfigParser.SafeConfigParser()
configSettings.read( configFileName )
configSection = "vipinfo"
self.host = configSettings.get( configSection, "host" )
self.port = configSettings.getint( configSection, "port" )
#landColor = configSettings.get( configSection, "color" )
#keep a copy of a background image around
#self.blank = PIL.Image.new('RGBA', (dims.METATILE*dims.TILE_SIZE, dims.METATILE*dims.TILE_SIZE), tuple(map(int, landColor.split(','))))
except Exception as detail:
mq_logging.error('Could not load terrain renderer configuration: %s' % (detail))
raise
def getTile( theURL ):
try:
#mq_logging.debug("getTile tile request %s" % ( theURL ))
theSocket = urllib.urlopen( theURL )
theData = theSocket.read()
theSocket.close()
#mq_logging.debug("getTile Received %d bytes from tile request %s" % ( len( theData ), theURL ))
if theSocket.getcode() != 200:
raise Exception(theSocket.getcode())
elif theData == 'No tile found':
raise Exception(theData)
else:
return theData
except Exception as detail:
mq_logging.error("getTile error fetching terrain image %s: %s" % ( theURL, str(detail) ))
return None
def handle_exception(request, exc_info):
if not isinstance(exc_info, tuple):
# Something is seriously wrong...
mq_logging.error(str(request))
mq_logging.error(str(exc_info))
raise SystemExit
mq_logging.error("Exception occured in request #%s: %s" % (request.requestID, exc_info))
def __init__(self, map_style, mask_style):
self.default_map = Map(0,0)
# only load the mask style if it's provided
if mask_style is None:
self.mask_map = None
else:
self.mask_map = Map(0,0)
self.proj = Mercator(18+1)
self.regions = []
try :
load_map(self.default_map, map_style)
# masks may not be provided if it's a single style
if self.mask_map is not None:
load_map(self.mask_map, mask_style)
self.map_proj = Projection(self.default_map.srs)
except:
mq_logging.error("Exception caught in Renderer ctor")
raise
def __init__ ( self, configFileName ):
self.concurrency = 16
try:
configSettings = ConfigParser.SafeConfigParser()
configSettings.read( configFileName )
#get the base url
self.url = configSettings.get('vipinfo', 'url').replace('$', '%')
self.concurrency = configSettings.getint('vipinfo', 'concurrency')
#validate the url as best we can
validated = self.url % {'z':'1', 'x':'2', 'y':'3'}
# one day there might be a parallel processing library for Python
# which works without problems and supports exceptions gracefully.
# today, i could not find it, if it exists...
#self.pool = Pool(self.concurrency)
except Exception as detail:
mq_logging.error('Could not load aerial/sat renderer configuration: %s' % (detail))
raise
def loadConfig(config):
try :
storage_conf = dict(config.items('storage'))
except Exception as ex:
mq_logging.error("failed to load worker configuration from: %s (%s)" % (config, str(ex)))
sys.exit(1)
storage = tile_storage.TileStorage(storage_conf)
formats = {}
format_args = {}
if config.has_option('worker','memory_limit_bytes'):
mem_limit = int(config.get('worker','memory_limit_bytes'))
else:
mem_limit = None
#load the coverages
coverageChecker = CoverageChecker(dict(config.items('coverages')))
#load the formats
formats = dict(config.items('formats'))
#parse out into a list
for coverage, csvFormats in formats.iteritems():
formats[coverage] = csv.reader([csvFormats], skipinitialspace=True).next()
# load settings for each of the data formats
for fmt in set(format for formatList in formats.values() for format in formatList):
try:
opts = dict(config.items(fmt))
# nasty hack, since PIL seems unwilling to coerce types and
# configparser gives everything back as strings.
if 'quality' in opts:
opts['quality'] = int(opts['quality'])
# again, nasty hack to allow the PIL name for a format to be
# different from the name in the config file.
if 'pil_name' not in opts:
opts['pil_name'] = fmt
format_args[fmt] = opts
except Exception as ex:
mq_logging.error("failed to load format configuration for '%s': %s" % (fmt, str(ex)))
sys.exit(1)
try:
renderers = RendererFactory(config, formats, format_args, storage)
except Exception as ex:
mq_logging.error("failed to load renderer configuration from: %s (%s)" % (config, str(ex)))
sys.exit(1)
#hand them all back
return storage, renderers, formats, format_args, coverageChecker, mem_limit
sys.exit(1)
#hand them all back
return storage, renderers, formats, format_args, coverageChecker, mem_limit
if __name__ == "__main__" :
option_parser = OptionParser(usage="usage: %prog [options] <worker-config> <queue-config> [<worker_id>]")
#option_parser.add_option("-h", "--help", dest="help", action="store_true", help="Print this helpful message.")
option_parser.add_option("-l", "--logging-config", dest="logging_config",
help="Path to configuration file for logging.")
(options, args) = option_parser.parse_args()
if len(args) != 2 and len(args) != 3:
mq_logging.error("Wrong number of command line arguments.")
option_parser.print_help()
sys.exit(1)
Watcher()
if options.logging_config:
log_config = ConfigParser()
mq_logging.configure_file(options.logging_config)
config = ConfigParser()
config.read(args[0])
#load the items from the config
storage, renderers, formats, format_args, coverageChecker, mem_limit = loadConfig(config)
sys.exit(1)
#hand them all back
return storage, renderers, formats, format_args, mem_limit
if __name__ == "__main__" :
option_parser = OptionParser(usage="usage: %prog [options] <worker-config> <queue-config> [<worker_id>]")
#option_parser.add_option("-h", "--help", dest="help", action="store_true", help="Print this helpful message.")
option_parser.add_option("-l", "--logging-config", dest="logging_config",
help="Path to configuration file for logging.")
(options, args) = option_parser.parse_args()
if len(args) != 2 and len(args) != 3:
mq_logging.error("Wrong number of command line arguments.")
option_parser.print_help()
sys.exit(1)
Watcher()
if options.logging_config:
log_config = ConfigParser()
mq_logging.configure_file(options.logging_config)
config = ConfigParser()
config.read(args[0])
#load the items from the config
storage, renderers, formats, format_args, mem_limit = loadConfig(config)