def open(self, url, data=None):
assert data is None, 'POST requests not supported by CGIClient'
parsed_url = urlparse(url)
environ = os.environ.copy()
environ.update({
'QUERY_STRING': parsed_url.query,
'REQUEST_METHOD': 'GET',
'GATEWAY_INTERFACE': 'CGI/1.1',
'SERVER_ADDR': '127.0.0.1',
'SERVER_NAME': 'localhost',
'SERVER_PROTOCOL': 'HTTP/1.0',
'SERVER_SOFTWARE': 'MapProxy',
})
start_time = time.time()
try:
p = subprocess.Popen([self.script], env=environ,
stdout=subprocess.PIPE,
cwd=self.working_directory or os.path.dirname(self.script)
)
except OSError, ex:
if ex.errno == errno.ENOENT:
raise SourceError('CGI script not found (%s)' % (self.script,))
elif ex.errno == errno.EACCES:
raise SourceError('No permission for CGI script (%s)' % (self.script,))
else:
raise
def open(self, url, data=None):
assert data is None, 'POST requests not supported by CGIClient'
parsed_url = urlparse.urlparse(url)
environ = os.environ.copy()
environ.update({
'QUERY_STRING': parsed_url.query,
'REQUEST_METHOD': 'GET',
'GATEWAY_INTERFACE': 'CGI/1.1',
'SERVER_ADDR': '127.0.0.1',
'SERVER_NAME': 'localhost',
'SERVER_PROTOCOL': 'HTTP/1.0',
'SERVER_SOFTWARE': 'MapProxy',
})
start_time = time.time()
try:
p = subprocess.Popen([self.script],
env=environ,
stdout=subprocess.PIPE,
cwd=self.working_directory
or os.path.dirname(self.script))
except OSError as ex:
if ex.errno == errno.ENOENT:
raise SourceError('CGI script not found (%s)' %
(self.script, ))
elif ex.errno == errno.EACCES:
raise SourceError('No permission for CGI script (%s)' %
(self.script, ))
else:
raise
stdout = p.communicate()[0]
ret = p.wait()
if ret != 0:
raise HTTPClientError('Error during CGI call (exit code: %d)' %
(ret, ))
if self.no_headers:
content = stdout
headers = dict()
else:
headers, content = split_cgi_response(stdout)
status_match = re.match('(\d\d\d) ', headers.get('Status', ''))
if status_match:
status_code = status_match.group(1)
else:
status_code = '-'
size = len(content)
content = IOwithHeaders(BytesIO(content), headers)
log_request('%s:%s' % (self.script, parsed_url.query),
status_code,
size=size,
method='CGI',
duration=time.time() - start_time)
return content
def _create_renderd_tile(self, tile_coord):
start_time = time.time()
result = self._send_tile_request(self.tile_mgr.identifier,
[tile_coord])
duration = time.time() - start_time
address = '%s:%s:%r' % (self.renderd_address, self.tile_mgr.identifier,
tile_coord)
if result['status'] == 'error':
log_request(address,
500,
None,
duration=duration,
method='RENDERD')
raise SourceError(
"Error from renderd: %s" %
result.get('error_message', 'unknown error from renderd'))
elif result['status'] == 'lock':
log_request(address,
503,
None,
duration=duration,
method='RENDERD')
raise LockTimeout("Lock timeout from renderd: %s" % result.get(
'error_message', 'unknown lock timeout error from renderd'))
log_request(address, 200, None, duration=duration, method='RENDERD')
def _check_resp(self, resp):
if 'Content-type' not in resp.headers:
raise SourceError(
'response from source WMS has no Content-type header')
if not resp.headers['Content-type'].startswith('image/'):
# log response depending on content-type
if resp.headers['Content-type'].startswith(
('text/', 'application/vnd.ogc')):
log_size = 8000 # larger xml exception
else:
log_size = 100 # image?
data = resp.read(log_size)
if len(data) == log_size:
data += '... truncated'
log.warn("expected image response, got: %s", data)
raise SourceError('no image returned from source WMS')
def _send_tile_request(self, cache_identifier, tile_coords):
identifier = hashlib.sha1(str((cache_identifier, tile_coords)).encode('ascii')).hexdigest()
message = {
'command': 'tile',
'id': identifier,
'tiles': tile_coords,
'cache_identifier': cache_identifier,
'priority': self.priority
}
try:
resp = requests.post(self.renderd_address, data=json.dumps(message))
return resp.json()
except ValueError:
raise SourceError("Error while communicating with renderd: invalid JSON")
except requests.RequestException as ex:
raise SourceError("Error while communicating with renderd: %s" % ex)
def _check_resp(self, resp, url):
if not resp.headers.get('Content-type', 'image/').startswith('image/'):
# log response depending on content-type
if resp.headers['Content-type'].startswith(
('text/', 'application/vnd.ogc')):
log_size = 8000 # larger xml exception
else:
log_size = 100 # image?
data = resp.read(log_size + 1)
truncated = ''
if len(data) == log_size + 1:
data = data[:-1]
truncated = ' [output truncated]'
if sys.version_info >= (3, 5, 0):
data = data.decode('utf-8', 'backslashreplace')
else:
data = data.decode('ascii', 'ignore')
log.warn(
"no image returned from source WMS: {}, response was: '{}'{}".
format(url, data, truncated))
raise SourceError('no image returned from source WMS: %s' %
(url, ))
def _check_resp(self, resp, url):
if not resp.headers.get('Content-type', 'image/').startswith('image/'):
# log response depending on content-type
if resp.headers['Content-type'].startswith(('text/', 'application/vnd.ogc')):
log_size = 8000 # larger xml exception
else:
log_size = 100 # image?
data = resp.read(log_size)
if len(data) == log_size:
data += '... truncated'
log.warn("no image returned from source WMS: %s, response was: %s" % (url, data))
raise SourceError('no image returned from source WMS: %s' % (url, ))
def is_cached(self, tile):
if tile.coord is None or tile.source:
return True
url = self.document_url(tile.coord)
resp = requests.get(url)
if resp.status_code == 200:
doc = json.loads(resp.content)
tile.timestamp = doc.get(self.md_template.timestamp_key)
return True
if resp.status_code == 404:
return False
raise SourceError('%r: %r' % (resp.status_code, resp.content))
def get_map(self, query):
if self.res_range and not self.res_range.contains(query.bbox, query.size,
query.srs):
raise BlankImage()
if self.coverage and not self.coverage.intersects(query.bbox, query.srs):
raise BlankImage()
try:
resp = self.render(query)
except RuntimeError, ex:
log.error('could not render Mapnik map: %s', ex)
reraise_exception(SourceError(ex.args[0]), sys.exc_info())
def _image(self, query):
try:
src_bbox, tile_grid, affected_tile_coords = \
self.grid.get_affected_tiles(query.bbox, query.size,
req_srs=query.srs)
except NoTiles:
raise BlankImage()
except GridError as ex:
raise MapBBOXError(ex.args[0])
num_tiles = tile_grid[0] * tile_grid[1]
if self.max_tile_limit and num_tiles >= self.max_tile_limit:
raise MapBBOXError(
"too many tiles, max_tile_limit: %s, num_tiles: %s" %
(self.max_tile_limit, num_tiles))
if query.tiled_only:
if num_tiles > 1:
raise MapBBOXError("not a single tile")
bbox = query.bbox
if not bbox_equals(bbox, src_bbox,
abs((bbox[2] - bbox[0]) / query.size[0] / 10),
abs((bbox[3] - bbox[1]) / query.size[1] / 10)):
raise MapBBOXError("query does not align to tile boundaries")
with self.tile_manager.session():
tile_collection = self.tile_manager.load_tile_coords(
affected_tile_coords, with_metadata=query.tiled_only)
if tile_collection.empty:
raise BlankImage()
if query.tiled_only:
tile = tile_collection[0].source
tile.image_opts = self.tile_manager.image_opts
tile.cacheable = tile_collection[0].cacheable
return tile
tile_sources = [tile.source for tile in tile_collection]
tiled_image = TiledImage(tile_sources,
src_bbox=src_bbox,
src_srs=self.grid.srs,
tile_grid=tile_grid,
tile_size=self.grid.tile_size)
try:
return tiled_image.transform(query.bbox, query.srs, query.size,
self.tile_manager.image_opts)
except ProjError:
raise MapBBOXError("could not transform query BBOX")
except IOError as ex:
from mapproxy.source import SourceError
raise SourceError("unable to transform image: %s" % ex)
def get_map(self, query):
if self.res_range and not self.res_range.contains(query.bbox, query.size,
query.srs):
raise BlankImage()
if self.coverage and not self.coverage.intersects(query.bbox, query.srs):
raise BlankImage()
try:
resp = self._get_map(query)
if self.transparent_color:
resp = make_transparent(resp, self.transparent_color,
self.transparent_color_tolerance)
resp.opacity = self.opacity
return resp
except HTTPClientError, e:
log.warn('could not retrieve WMS map: %s', e)
reraise_exception(SourceError(e.args[0]), sys.exc_info())
def is_cached(self, tile):
if tile.coord is None or tile.source:
return True
url = self.document_url(tile.coord)
try:
self.init_db()
resp = self.req_session.get(url)
if resp.status_code == 200:
doc = json.loads(codecs.decode(resp.content, 'utf-8'))
tile.timestamp = doc.get(self.md_template.timestamp_key)
return True
except (requests.exceptions.RequestException, socket.error) as ex:
# is_cached should not fail (would abort seeding for example),
# so we catch these errors here and just return False
log.warn('error while requesting %s: %s', url, ex)
return False
if resp.status_code == 404:
return False
raise SourceError('%r: %r' % (resp.status_code, resp.content))
def get_map(self, query):
if self.grid.tile_size != query.size:
ex = InvalidSourceQuery(
'tile size of cache and tile source do not match: %s != %s' %
(self.grid.tile_size, query.size))
log_config.error(ex)
raise ex
if self.grid.srs != query.srs:
ex = InvalidSourceQuery(
'SRS of cache and tile source do not match: %r != %r' %
(self.grid.srs, query.srs))
log_config.error(ex)
raise ex
if self.res_range and not self.res_range.contains(
query.bbox, query.size, query.srs):
raise BlankImage()
if self.coverage and not self.coverage.intersects(
query.bbox, query.srs):
raise BlankImage()
_bbox, grid, tiles = self.grid.get_affected_tiles(
query.bbox, query.size)
if grid != (1, 1):
raise InvalidSourceQuery('BBOX does not align to tile')
tile_coord = next(tiles)
try:
return self.client.get_tile(tile_coord, format=query.format)
except HTTPClientError as e:
if self.error_handler:
resp = self.error_handler.handle(e.response_code, query)
if resp:
return resp
log.warn('could not retrieve tile: %s', e)
reraise_exception(SourceError(e.args[0]), sys.exc_info())
class CacheMapLayer(MapLayer):
supports_meta_tiles = True
def __init__(self, tile_manager, extent=None, image_opts=None,
max_tile_limit=None):
MapLayer.__init__(self, image_opts=image_opts)
self.tile_manager = tile_manager
self.grid = tile_manager.grid
self.extent = extent or map_extent_from_grid(self.grid)
self.res_range = merge_layer_res_ranges(self.tile_manager.sources)
self.transparent = tile_manager.transparent
self.max_tile_limit = max_tile_limit
def get_map(self, query):
self.check_res_range(query)
if query.tiled_only:
self._check_tiled(query)
query_extent = MapExtent(query.bbox, query.srs)
if not query.tiled_only and self.extent and not self.extent.contains(query_extent):
if not self.extent.intersects(query_extent):
raise BlankImage()
size, offset, bbox = bbox_position_in_image(query.bbox, query.size, self.extent.bbox_for(query.srs))
if size[0] == 0 or size[1] == 0:
raise BlankImage()
src_query = MapQuery(bbox, size, query.srs, query.format)
resp = self._image(src_query)
result = SubImageSource(resp, size=query.size, offset=offset, image_opts=self.image_opts,
cacheable=resp.cacheable)
else:
result = self._image(query)
return result
def _check_tiled(self, query):
if query.format != self.tile_manager.format:
raise MapError("invalid tile format, use %s" % self.tile_manager.format)
if query.size != self.grid.tile_size:
raise MapError("invalid tile size (use %dx%d)" % self.grid.tile_size)
def _image(self, query):
try:
src_bbox, tile_grid, affected_tile_coords = \
self.grid.get_affected_tiles(query.bbox, query.size,
req_srs=query.srs)
except NoTiles:
raise BlankImage()
except GridError, ex:
raise MapBBOXError(ex.args[0])
num_tiles = tile_grid[0] * tile_grid[1]
if self.max_tile_limit and num_tiles >= self.max_tile_limit:
raise MapBBOXError("too many tiles")
if query.tiled_only:
if num_tiles > 1:
raise MapBBOXError("not a single tile")
bbox = query.bbox
if not bbox_equals(bbox, src_bbox, abs((bbox[2]-bbox[0])/query.size[0]/10),
abs((bbox[3]-bbox[1])/query.size[1]/10)):
raise MapBBOXError("query does not align to tile boundaries")
with self.tile_manager.session():
tile_collection = self.tile_manager.load_tile_coords(affected_tile_coords, with_metadata=query.tiled_only)
if tile_collection.empty:
raise BlankImage()
if query.tiled_only:
tile = tile_collection[0].source
tile.image_opts = self.tile_manager.image_opts
tile.cacheable = tile_collection[0].cacheable
return tile
tile_sources = [tile.source for tile in tile_collection]
tiled_image = TiledImage(tile_sources, src_bbox=src_bbox, src_srs=self.grid.srs,
tile_grid=tile_grid, tile_size=self.grid.tile_size)
try:
return tiled_image.transform(query.bbox, query.srs, query.size,
self.tile_manager.image_opts)
except ProjError:
raise MapBBOXError("could not transform query BBOX")
except IOError, ex:
from mapproxy.source import SourceError
raise SourceError("unable to transform image: %s" % ex)
def _check_resp(self, resp):
if not resp.headers.get('Content-type', 'image/').startswith('image/'):
raise SourceError('no image returned from static LegendURL')
url = self.document_url(tile.coord)
try:
self.init_db()
resp = self.req_session.get(url)
if resp.status_code == 200:
doc = json.loads(resp.content)
tile.timestamp = doc.get(self.md_template.timestamp_key)
return True
except (requests.exceptions.RequestException, socket.error), ex:
# is_cached should not fail (would abort seeding for example),
# so we catch these errors here and just return False
log.warn('error while requesting %s: %s', url, ex)
return False
if resp.status_code == 404:
return False
raise SourceError('%r: %r' % (resp.status_code, resp.content))
def _tile_doc(self, tile):
tile_id = self.document_url(tile.coord, relative=True)
if self.md_template:
tile_doc = self.md_template.doc(tile, self.tile_grid)
else:
tile_doc = {}
tile_doc['_id'] = tile_id
with tile_buffer(tile) as buf:
data = buf.read()
tile_doc['_attachments'] = {
'tile': {
'content_type': 'image/' + self.file_ext,
'data': data.encode('base64').replace('\n', ''),
def _check_resp(self, resp):
if 'Content-type' not in resp.headers:
raise SourceError(
'response from source WMS has no Content-type header')
if not resp.headers['Content-type'].startswith('image/'):
raise SourceError('no image returned from static LegendURL')
