def open(self, url, data=None): assert data is None, 'POST requests not supported by CGIClient' parsed_url = urlparse(url) environ = os.environ.copy() environ.update({ 'QUERY_STRING': parsed_url.query, 'REQUEST_METHOD': 'GET', 'GATEWAY_INTERFACE': 'CGI/1.1', 'SERVER_ADDR': '127.0.0.1', 'SERVER_NAME': 'localhost', 'SERVER_PROTOCOL': 'HTTP/1.0', 'SERVER_SOFTWARE': 'MapProxy', }) start_time = time.time() try: p = subprocess.Popen([self.script], env=environ, stdout=subprocess.PIPE, cwd=self.working_directory or os.path.dirname(self.script) ) except OSError, ex: if ex.errno == errno.ENOENT: raise SourceError('CGI script not found (%s)' % (self.script,)) elif ex.errno == errno.EACCES: raise SourceError('No permission for CGI script (%s)' % (self.script,)) else: raise
def open(self, url, data=None): assert data is None, 'POST requests not supported by CGIClient' parsed_url = urlparse.urlparse(url) environ = os.environ.copy() environ.update({ 'QUERY_STRING': parsed_url.query, 'REQUEST_METHOD': 'GET', 'GATEWAY_INTERFACE': 'CGI/1.1', 'SERVER_ADDR': '127.0.0.1', 'SERVER_NAME': 'localhost', 'SERVER_PROTOCOL': 'HTTP/1.0', 'SERVER_SOFTWARE': 'MapProxy', }) start_time = time.time() try: p = subprocess.Popen([self.script], env=environ, stdout=subprocess.PIPE, cwd=self.working_directory or os.path.dirname(self.script)) except OSError as ex: if ex.errno == errno.ENOENT: raise SourceError('CGI script not found (%s)' % (self.script, )) elif ex.errno == errno.EACCES: raise SourceError('No permission for CGI script (%s)' % (self.script, )) else: raise stdout = p.communicate()[0] ret = p.wait() if ret != 0: raise HTTPClientError('Error during CGI call (exit code: %d)' % (ret, )) if self.no_headers: content = stdout headers = dict() else: headers, content = split_cgi_response(stdout) status_match = re.match('(\d\d\d) ', headers.get('Status', '')) if status_match: status_code = status_match.group(1) else: status_code = '-' size = len(content) content = IOwithHeaders(BytesIO(content), headers) log_request('%s:%s' % (self.script, parsed_url.query), status_code, size=size, method='CGI', duration=time.time() - start_time) return content
def _create_renderd_tile(self, tile_coord): start_time = time.time() result = self._send_tile_request(self.tile_mgr.identifier, [tile_coord]) duration = time.time() - start_time address = '%s:%s:%r' % (self.renderd_address, self.tile_mgr.identifier, tile_coord) if result['status'] == 'error': log_request(address, 500, None, duration=duration, method='RENDERD') raise SourceError( "Error from renderd: %s" % result.get('error_message', 'unknown error from renderd')) elif result['status'] == 'lock': log_request(address, 503, None, duration=duration, method='RENDERD') raise LockTimeout("Lock timeout from renderd: %s" % result.get( 'error_message', 'unknown lock timeout error from renderd')) log_request(address, 200, None, duration=duration, method='RENDERD')
def _check_resp(self, resp): if 'Content-type' not in resp.headers: raise SourceError( 'response from source WMS has no Content-type header') if not resp.headers['Content-type'].startswith('image/'): # log response depending on content-type if resp.headers['Content-type'].startswith( ('text/', 'application/vnd.ogc')): log_size = 8000 # larger xml exception else: log_size = 100 # image? data = resp.read(log_size) if len(data) == log_size: data += '... truncated' log.warn("expected image response, got: %s", data) raise SourceError('no image returned from source WMS')
def _send_tile_request(self, cache_identifier, tile_coords): identifier = hashlib.sha1(str((cache_identifier, tile_coords)).encode('ascii')).hexdigest() message = { 'command': 'tile', 'id': identifier, 'tiles': tile_coords, 'cache_identifier': cache_identifier, 'priority': self.priority } try: resp = requests.post(self.renderd_address, data=json.dumps(message)) return resp.json() except ValueError: raise SourceError("Error while communicating with renderd: invalid JSON") except requests.RequestException as ex: raise SourceError("Error while communicating with renderd: %s" % ex)
def _check_resp(self, resp, url): if not resp.headers.get('Content-type', 'image/').startswith('image/'): # log response depending on content-type if resp.headers['Content-type'].startswith( ('text/', 'application/vnd.ogc')): log_size = 8000 # larger xml exception else: log_size = 100 # image? data = resp.read(log_size + 1) truncated = '' if len(data) == log_size + 1: data = data[:-1] truncated = ' [output truncated]' if sys.version_info >= (3, 5, 0): data = data.decode('utf-8', 'backslashreplace') else: data = data.decode('ascii', 'ignore') log.warn( "no image returned from source WMS: {}, response was: '{}'{}". format(url, data, truncated)) raise SourceError('no image returned from source WMS: %s' % (url, ))
def _check_resp(self, resp, url): if not resp.headers.get('Content-type', 'image/').startswith('image/'): # log response depending on content-type if resp.headers['Content-type'].startswith(('text/', 'application/vnd.ogc')): log_size = 8000 # larger xml exception else: log_size = 100 # image? data = resp.read(log_size) if len(data) == log_size: data += '... truncated' log.warn("no image returned from source WMS: %s, response was: %s" % (url, data)) raise SourceError('no image returned from source WMS: %s' % (url, ))
def is_cached(self, tile): if tile.coord is None or tile.source: return True url = self.document_url(tile.coord) resp = requests.get(url) if resp.status_code == 200: doc = json.loads(resp.content) tile.timestamp = doc.get(self.md_template.timestamp_key) return True if resp.status_code == 404: return False raise SourceError('%r: %r' % (resp.status_code, resp.content))
def get_map(self, query): if self.res_range and not self.res_range.contains(query.bbox, query.size, query.srs): raise BlankImage() if self.coverage and not self.coverage.intersects(query.bbox, query.srs): raise BlankImage() try: resp = self.render(query) except RuntimeError, ex: log.error('could not render Mapnik map: %s', ex) reraise_exception(SourceError(ex.args[0]), sys.exc_info())
def _image(self, query): try: src_bbox, tile_grid, affected_tile_coords = \ self.grid.get_affected_tiles(query.bbox, query.size, req_srs=query.srs) except NoTiles: raise BlankImage() except GridError as ex: raise MapBBOXError(ex.args[0]) num_tiles = tile_grid[0] * tile_grid[1] if self.max_tile_limit and num_tiles >= self.max_tile_limit: raise MapBBOXError( "too many tiles, max_tile_limit: %s, num_tiles: %s" % (self.max_tile_limit, num_tiles)) if query.tiled_only: if num_tiles > 1: raise MapBBOXError("not a single tile") bbox = query.bbox if not bbox_equals(bbox, src_bbox, abs((bbox[2] - bbox[0]) / query.size[0] / 10), abs((bbox[3] - bbox[1]) / query.size[1] / 10)): raise MapBBOXError("query does not align to tile boundaries") with self.tile_manager.session(): tile_collection = self.tile_manager.load_tile_coords( affected_tile_coords, with_metadata=query.tiled_only) if tile_collection.empty: raise BlankImage() if query.tiled_only: tile = tile_collection[0].source tile.image_opts = self.tile_manager.image_opts tile.cacheable = tile_collection[0].cacheable return tile tile_sources = [tile.source for tile in tile_collection] tiled_image = TiledImage(tile_sources, src_bbox=src_bbox, src_srs=self.grid.srs, tile_grid=tile_grid, tile_size=self.grid.tile_size) try: return tiled_image.transform(query.bbox, query.srs, query.size, self.tile_manager.image_opts) except ProjError: raise MapBBOXError("could not transform query BBOX") except IOError as ex: from mapproxy.source import SourceError raise SourceError("unable to transform image: %s" % ex)
def get_map(self, query): if self.res_range and not self.res_range.contains(query.bbox, query.size, query.srs): raise BlankImage() if self.coverage and not self.coverage.intersects(query.bbox, query.srs): raise BlankImage() try: resp = self._get_map(query) if self.transparent_color: resp = make_transparent(resp, self.transparent_color, self.transparent_color_tolerance) resp.opacity = self.opacity return resp except HTTPClientError, e: log.warn('could not retrieve WMS map: %s', e) reraise_exception(SourceError(e.args[0]), sys.exc_info())
def is_cached(self, tile): if tile.coord is None or tile.source: return True url = self.document_url(tile.coord) try: self.init_db() resp = self.req_session.get(url) if resp.status_code == 200: doc = json.loads(codecs.decode(resp.content, 'utf-8')) tile.timestamp = doc.get(self.md_template.timestamp_key) return True except (requests.exceptions.RequestException, socket.error) as ex: # is_cached should not fail (would abort seeding for example), # so we catch these errors here and just return False log.warn('error while requesting %s: %s', url, ex) return False if resp.status_code == 404: return False raise SourceError('%r: %r' % (resp.status_code, resp.content))
def get_map(self, query): if self.grid.tile_size != query.size: ex = InvalidSourceQuery( 'tile size of cache and tile source do not match: %s != %s' % (self.grid.tile_size, query.size)) log_config.error(ex) raise ex if self.grid.srs != query.srs: ex = InvalidSourceQuery( 'SRS of cache and tile source do not match: %r != %r' % (self.grid.srs, query.srs)) log_config.error(ex) raise ex if self.res_range and not self.res_range.contains( query.bbox, query.size, query.srs): raise BlankImage() if self.coverage and not self.coverage.intersects( query.bbox, query.srs): raise BlankImage() _bbox, grid, tiles = self.grid.get_affected_tiles( query.bbox, query.size) if grid != (1, 1): raise InvalidSourceQuery('BBOX does not align to tile') tile_coord = next(tiles) try: return self.client.get_tile(tile_coord, format=query.format) except HTTPClientError as e: if self.error_handler: resp = self.error_handler.handle(e.response_code, query) if resp: return resp log.warn('could not retrieve tile: %s', e) reraise_exception(SourceError(e.args[0]), sys.exc_info())
class CacheMapLayer(MapLayer): supports_meta_tiles = True def __init__(self, tile_manager, extent=None, image_opts=None, max_tile_limit=None): MapLayer.__init__(self, image_opts=image_opts) self.tile_manager = tile_manager self.grid = tile_manager.grid self.extent = extent or map_extent_from_grid(self.grid) self.res_range = merge_layer_res_ranges(self.tile_manager.sources) self.transparent = tile_manager.transparent self.max_tile_limit = max_tile_limit def get_map(self, query): self.check_res_range(query) if query.tiled_only: self._check_tiled(query) query_extent = MapExtent(query.bbox, query.srs) if not query.tiled_only and self.extent and not self.extent.contains(query_extent): if not self.extent.intersects(query_extent): raise BlankImage() size, offset, bbox = bbox_position_in_image(query.bbox, query.size, self.extent.bbox_for(query.srs)) if size[0] == 0 or size[1] == 0: raise BlankImage() src_query = MapQuery(bbox, size, query.srs, query.format) resp = self._image(src_query) result = SubImageSource(resp, size=query.size, offset=offset, image_opts=self.image_opts, cacheable=resp.cacheable) else: result = self._image(query) return result def _check_tiled(self, query): if query.format != self.tile_manager.format: raise MapError("invalid tile format, use %s" % self.tile_manager.format) if query.size != self.grid.tile_size: raise MapError("invalid tile size (use %dx%d)" % self.grid.tile_size) def _image(self, query): try: src_bbox, tile_grid, affected_tile_coords = \ self.grid.get_affected_tiles(query.bbox, query.size, req_srs=query.srs) except NoTiles: raise BlankImage() except GridError, ex: raise MapBBOXError(ex.args[0]) num_tiles = tile_grid[0] * tile_grid[1] if self.max_tile_limit and num_tiles >= self.max_tile_limit: raise MapBBOXError("too many tiles") if query.tiled_only: if num_tiles > 1: raise MapBBOXError("not a single tile") bbox = query.bbox if not bbox_equals(bbox, src_bbox, abs((bbox[2]-bbox[0])/query.size[0]/10), abs((bbox[3]-bbox[1])/query.size[1]/10)): raise MapBBOXError("query does not align to tile boundaries") with self.tile_manager.session(): tile_collection = self.tile_manager.load_tile_coords(affected_tile_coords, with_metadata=query.tiled_only) if tile_collection.empty: raise BlankImage() if query.tiled_only: tile = tile_collection[0].source tile.image_opts = self.tile_manager.image_opts tile.cacheable = tile_collection[0].cacheable return tile tile_sources = [tile.source for tile in tile_collection] tiled_image = TiledImage(tile_sources, src_bbox=src_bbox, src_srs=self.grid.srs, tile_grid=tile_grid, tile_size=self.grid.tile_size) try: return tiled_image.transform(query.bbox, query.srs, query.size, self.tile_manager.image_opts) except ProjError: raise MapBBOXError("could not transform query BBOX") except IOError, ex: from mapproxy.source import SourceError raise SourceError("unable to transform image: %s" % ex)
def _check_resp(self, resp): if not resp.headers.get('Content-type', 'image/').startswith('image/'): raise SourceError('no image returned from static LegendURL')
url = self.document_url(tile.coord) try: self.init_db() resp = self.req_session.get(url) if resp.status_code == 200: doc = json.loads(resp.content) tile.timestamp = doc.get(self.md_template.timestamp_key) return True except (requests.exceptions.RequestException, socket.error), ex: # is_cached should not fail (would abort seeding for example), # so we catch these errors here and just return False log.warn('error while requesting %s: %s', url, ex) return False if resp.status_code == 404: return False raise SourceError('%r: %r' % (resp.status_code, resp.content)) def _tile_doc(self, tile): tile_id = self.document_url(tile.coord, relative=True) if self.md_template: tile_doc = self.md_template.doc(tile, self.tile_grid) else: tile_doc = {} tile_doc['_id'] = tile_id with tile_buffer(tile) as buf: data = buf.read() tile_doc['_attachments'] = { 'tile': { 'content_type': 'image/' + self.file_ext, 'data': data.encode('base64').replace('\n', ''),
def _check_resp(self, resp): if 'Content-type' not in resp.headers: raise SourceError( 'response from source WMS has no Content-type header') if not resp.headers['Content-type'].startswith('image/'): raise SourceError('no image returned from static LegendURL')