def tilelist(data,zooms):
if (data["filetype"]):
coordinates = MBTiles.list_tiles(data["filename"])
else:
lat1, lon1, lat2, lon2 = data["bbox"]
south, west = min(lat1, lat2), min(lon1, lon2)
north, east = max(lat1, lat2), max(lon1, lon2)
northwest = Location(north, west)
southeast = Location(south, east)
osm = Provider()
ul = osm.locationCoordinate(northwest)
lr = osm.locationCoordinate(southeast)
for (i, zoom) in enumerate(zooms):
if not zoom.isdigit():
raise KnownUnknown('"%s" is not a valid numeric zoom level.' % zoom)
zooms[i] = int(zoom)
if data["padding"] < 0:
raise KnownUnknown('A negative padding will not work.')
coordinates = generateCoordinates(ul, lr, zooms, data["padding"])
return coordinates
def validate_bbox(self, bbox_dict):
if not (-90.0 < bbox_dict['south'] <
90.0) or not (-90.0 < bbox_dict['north'] < 90.0):
raise KnownUnknown(
'Latitude must be a value between -90 and 90 '
'(Hint: Maybe you did long/lat instead of lat/long?).')
if not (-180.0 < bbox_dict['west'] <
180.0) or not (-180.0 < bbox_dict['east'] < 180.0):
raise KnownUnknown(
'Longitude must be a value between -180 and 180.')
def _feature_properties(feature,
layer_definition,
whitelist=None,
skip_empty_fields=False):
""" Returns a dictionary of feature properties for a feature in a layer.
Third argument is an optional list or dictionary of properties to
whitelist by case-sensitive name - leave it None to include everything.
A dictionary will cause property names to be re-mapped.
OGR property types:
OFTInteger (0), OFTIntegerList (1), OFTReal (2), OFTRealList (3),
OFTString (4), OFTStringList (5), OFTWideString (6), OFTWideStringList (7),
OFTBinary (8), OFTDate (9), OFTTime (10), OFTDateTime (11).
Extra OGR types for GDAL 2.x:
OFTInteger64 (12), OFTInteger64List (13)
"""
properties = {}
okay_types = [
ogr.OFTInteger, ogr.OFTReal, ogr.OFTString, ogr.OFTWideString,
ogr.OFTDate, ogr.OFTTime, ogr.OFTDateTime
]
if hasattr(ogr, 'OFTInteger64'):
okay_types.extend([ogr.OFTInteger64, ogr.OFTInteger64List])
for index in range(layer_definition.GetFieldCount()):
field_definition = layer_definition.GetFieldDefn(index)
field_type = field_definition.GetType()
name = field_definition.GetNameRef()
if type(whitelist) in (list, dict) and name not in whitelist:
continue
if field_type not in okay_types:
try:
name = [
oft for oft in dir(ogr) if oft.startswith('OFT')
and getattr(ogr, oft) == field_type
][0]
except IndexError:
raise KnownUnknown(
"Found an OGR field type I've never even seen: %d" %
field_type)
else:
raise KnownUnknown(
"Found an OGR field type I don't know what to do with: ogr.%s"
% name)
if not skip_empty_fields or feature.IsFieldSet(name):
property = type(whitelist) is dict and whitelist[name] or name
properties[property] = feature.GetField(name)
return properties
def apply_adjustments(rgba, adjustments):
""" Apply image adjustments one by one and return a modified image.
Working adjustments:
threshold:
Calls apply_threshold_adjustment()
curves:
Calls apply_curves_adjustment()
curves2:
Calls apply_curves2_adjustment()
"""
if not adjustments:
return rgba
for adjustment in adjustments:
name, args = adjustment[0], adjustment[1:]
if name == 'threshold':
rgba = apply_threshold_adjustment(rgba, *args)
elif name == 'curves':
rgba = apply_curves_adjustment(rgba, *args)
elif name == 'curves2':
rgba = apply_curves2_adjustment(rgba, *args)
else:
raise KnownUnknown(
'Unrecognized composite adjustment: "%s" with args %s' %
(name, repr(args)))
return rgba
def save(self, out, format):
if format != 'JSON':
raise KnownUnknown('PostGeoJSON only saves .json tiles, not "%s"' %
format)
indent = None
if int(self.indent) > 0:
indent = self.indent
encoded = JSONEncoder(indent=indent).iterencode(self.content)
float_pat = compile(r'^-?\d+\.\d+$')
precision = 6
if int(self.precision) > 0:
precision = self.precision
format = '%.' + str(precision) + 'f'
for atom in encoded:
if float_pat.match(atom):
out.write(format % float(atom))
else:
out.write(atom)
def getTypeByExtension(self, extension):
""" Get mime-type and format by file extension.
This only accepts "geojson" for the time being.
"""
if extension.lower() == 'geojson':
return 'application/json', 'GeoJSON'
elif extension.lower() == 'arcjson':
return 'application/json', 'ArcJSON'
elif extension.lower() == 'geobson':
return 'application/x-bson', 'GeoBSON'
elif extension.lower() == 'arcbson':
return 'application/x-bson', 'ArcBSON'
elif extension.lower() == 'geoamf':
return 'application/x-amf', 'GeoAMF'
elif extension.lower() == 'arcamf':
return 'application/x-amf', 'ArcAMF'
elif extension.lower() == 'wkt':
return 'text/x-wkt', 'WKT'
raise KnownUnknown('Vector Provider only makes .geojson, .arcjson, .geobson, .arcbson, .geoamf, .arcamf and .wkt tiles, not "%s"' % extension)
def getTypeByExtension(self, extension):
if extension.lower() == 'png':
return 'image/png', 'PNG'
raise KnownUnknown('I only know how to make .png tiles, not "%s"' %
extension)
def save(self, out, format):
if format != self.format:
raise KnownUnknown(
'Monkeycache only knows how to make %s tiles, not %s' %
(self.format, format))
out.write(self.body)
def save(self, out, format):
if format != 'JSON':
raise KnownUnknown('MapnikGrid only saves .json tiles, not "%s"' %
format)
bytes = json.dumps(self.content, ensure_ascii=False).encode('utf-8')
out.write(bytes)
def save(self, out, format):
if format not in ('PNG'):
raise KnownUnknown('We only saves .png tiles, not "%s"' % format)
if not self.data['has_stuff']:
im = Image.new('RGBA', (self.width, self.height))
im.save(out, 'PNG')
return
fh, tmpfile = tempfile.mkstemp('.json')
os.write(fh, json.dumps(self.data['geojson']))
os.close(fh)
map = mapnik.Map(0, 0)
map.srs = '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs'
datasource = mapnik.Ogr(base=os.path.dirname(tmpfile),
file=os.path.basename(tmpfile),
layer='OGRGeoJSON')
os.unlink(tmpfile)
lyr = mapnik.Layer('xapi_raster')
lyr.srs = '+proj=latlong +datum=WGS84'
lyr.datasource = datasource
style = mapnik.Style()
rule = mapnik.Rule()
if self.data['filltype'] == 'line':
fill = mapnik.Color(str(self.data['fill']))
# TO DO: make me a config flag
# rule.symbols.append(mapnik.LineSymbolizer(fill, 1.0))
rule.symbols.append(mapnik.LineSymbolizer(fill, 3.0))
style.rules.append(rule)
else:
fill = mapnik.Color(str(self.data['fill']))
rule.symbols.append(mapnik.PolygonSymbolizer(fill))
style.rules.append(rule)
map.append_style('xapi_raster', style)
lyr.styles.append('xapi_raster')
map.layers.append(lyr)
xmin, ymin, xmax, ymax = self.data['bbox']
env = mapnik.Envelope(xmin, ymin, xmax, ymax)
map.width = self.width
map.height = self.height
map.zoom_to_box(env)
img = mapnik.Image(self.width, self.height)
mapnik.render(map, img)
img = Image.fromstring('RGBA', (self.width, self.height),
img.tostring())
img.save(out, 'PNG')
def getTypeByExtension(self, extension):
""" Get mime-type and format by file extension.
This only accepts "json".
"""
if extension.lower() != 'json':
raise KnownUnknown('UtfGridComposite only makes .json tiles, not "%s"' % extension)
return 'text/json', 'JSON'
def getTypeByExtension(self, extension):
""" Get mime-type and format by file extension.
This only accepts "xml".
"""
if extension.lower() != 'xml':
raise KnownUnknown('TileDataOSM only makes .xml tiles, not "%s"' % extension)
return 'text/xml', 'XML'
def getTypeByExtension(self, extension):
""" Get mime-type and format by file extension.
This only accepts "json".
"""
if extension.lower() != 'json':
raise KnownUnknown('MapnikGrid only makes .json tiles, not "%s"' % extension)
return 'application/json; charset=utf-8', 'JSON'
def save(self, out, format):
"""
"""
#
# Serialize
#
if format == 'WKT':
if 'wkt' in self.content['crs']:
out.write(self.content['crs']['wkt'])
else:
out.write(_sref_4326().ExportToWkt())
return
if format in ('GeoJSON', 'GeoBSON', 'GeoAMF'):
content = self.content
if 'wkt' in content['crs']:
content['crs'] = {'type': 'link', 'properties': {'href': '0.wkt', 'type': 'ogcwkt'}}
else:
del content['crs']
elif format in ('ArcJSON', 'ArcBSON', 'ArcAMF'):
content = reserialize_to_arc(self.content, format == 'ArcAMF')
else:
raise KnownUnknown('Vector response only saves .geojson, .arcjson, .geobson, .arcbson, .geoamf, .arcamf and .wkt tiles, not "%s"' % format)
#
# Encode
#
if format in ('GeoJSON', 'ArcJSON'):
indent = self.verbose and 2 or None
encoded = JSONEncoder(indent=indent).iterencode(content)
float_pat = compile(r'^-?\d+\.\d+$')
for atom in encoded:
if float_pat.match(atom):
out.write(('%%.%if' % self.precision) % float(atom))
else:
out.write(atom)
elif format in ('GeoBSON', 'ArcBSON'):
import bson
encoded = bson.dumps(content)
out.write(encoded)
elif format in ('GeoAMF', 'ArcAMF'):
import pyamf
for class_name in pyamf_classes.items():
pyamf.register_class(*class_name)
encoded = pyamf.encode(content, 0).read()
out.write(encoded)
def getTypeByExtension(self, extension):
""" Return (image/jpeg, JPEG) for extension "jpg", throw error for anything else.
"""
if extension == 'jpg':
return ('image/jpeg', 'JPEG')
raise KnownUnknown(
'oam.tiles.Provider only wants to make "jpg" tiles, not "%s"' %
extension)
def getTypeByExtension(self, extension):
""" Get mime-type and format by file extension.
This only accepts "vtm" for the time being.
"""
if extension.lower() == 'vtm':
return 'image/png', 'VTM'
#return 'application/x-protobuf', 'VTM'
raise KnownUnknown('OpenScienceMap Provider only makes ".vtm", not "%s"' % extension)
def save(self, out, format):
if format == 'TXT':
out.write(self.content)
elif format == 'PNG':
image = self.do_I_have_to_draw_you_a_picture()
image.save(out, format)
else:
raise KnownUnknown('MirrorOSM only saves .txt and .png tiles, not "%s"' % format)
def getTypeByExtension(self, extension):
'''
'''
if extension == 'big':
return 'application/octet-stream', 'Big Endian'
if extension == 'little':
return 'application/octet-stream', 'Little Endian'
raise KnownUnknown('Unknown type: "%s".' % extension)
def renderTile(self, width, height, srs, coord):
""" Render a single tile, return a ConfirmationResponse instance.
"""
if coord.zoom < 12:
raise KnownUnknown(
'MirrorOSM provider only handles data at zoom 12 or higher, not %d.'
% coord.zoom)
start = time()
garbage = []
handle, filename = mkstemp(prefix='mirrorosm-', suffix='.tablename')
tmp_prefix = 'mirrorosm_' + b16encode(
basename(filename)[10:-10]).lower()
garbage.append(filename)
close(handle)
handle, filename = mkstemp(prefix='mirrorosm-', suffix='.osm.gz')
garbage.append(filename)
close(handle)
try:
length = download_api_data(filename, coord, self.api_base,
self.layer.projection)
prepare_data(filename, tmp_prefix, self.dbkwargs, self.osm2pgsql,
self.layer.projection)
db = _connect(**self.dbkwargs).cursor()
ul = self.layer.projection.coordinateProj(coord)
lr = self.layer.projection.coordinateProj(coord.down().right())
create_tables(db, self.prefix, tmp_prefix)
populate_tables(db, self.prefix, tmp_prefix,
(ul.x, ul.y, lr.x, lr.y))
clean_up_tables(db, tmp_prefix)
db.close()
message = 'Retrieved %dK of OpenStreetMap data for tile %d/%d/%d in %.2fsec from %s (%s).\n' \
% (length, coord.zoom, coord.column, coord.row,
(time() - start), self.api_base, datetime.now())
return ConfirmationResponse(coord, message, True)
except Exception as e:
message = 'Error in tile %d/%d/%d: %s' % (coord.zoom, coord.column,
coord.row, e)
raise NoTileLeftBehind(ConfirmationResponse(coord, message, False))
finally:
for filename in garbage:
unlink(filename)
def save(self, out, format):
if format != 'JSON':
raise KnownUnknown('SolrGeoJSON only saves .json tiles, not "%s"' % format)
encoded = JSONEncoder(indent=2).iterencode(self.content)
float_pat = compile(r'^-?\d+\.\d+$')
for atom in encoded:
if float_pat.match(atom):
out.write('%.6f' % float(atom))
else:
out.write(atom)
def make_color(color):
""" Convert colors expressed as HTML-style RGB(A) strings to tuples.
Returns four-element RGBA tuple, e.g. (0xFF, 0x99, 0x00, 0xFF).
Examples:
white: "#ffffff", "#fff", "#ffff", "#ffffffff"
black: "#000000", "#000", "#000f", "#000000ff"
null: "#0000", "#00000000"
orange: "#f90", "#ff9900", "#ff9900ff"
transparent orange: "#f908", "#ff990088"
"""
if type(color) not in (str, unicode):
raise KnownUnknown('Color must be a string: %s' % repr(color))
if color[0] != '#':
raise KnownUnknown('Color must start with hash: "%s"' % color)
if len(color) not in (4, 5, 7, 9):
raise KnownUnknown(
'Color must have three, four, six or seven hex chars: "%s"' %
color)
if len(color) == 4:
color = ''.join([color[i] for i in (0, 1, 1, 2, 2, 3, 3)])
elif len(color) == 5:
color = ''.join([color[i] for i in (0, 1, 1, 2, 2, 3, 3, 4, 4)])
try:
r = int(color[1:3], 16)
g = int(color[3:5], 16)
b = int(color[5:7], 16)
a = len(color) == 7 and 0xFF or int(color[7:9], 16)
except ValueError:
raise KnownUnknown('Color must be made up of valid hex chars: "%s"' %
color)
return r, g, b, a
def save(self, out, format):
if format != 'VTM':
raise KnownUnknown('OpenScienceMap provider only saves .vtm tiles, not "%s"' % format)
data = self.content.SerializeToString()
# zbuf = cStringIO.StringIO()
# zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=9)
# zfile.write(data)
# zout = zlib.compress(data, 9)
# logging.debug("serialized %s - %fkb <> %fkb" %(self.tile, len(data)/1024.0, len(zout)/1024.0))
out.write(struct.pack(">I", len(data)))
out.write(data)
def getTypeByExtension(self, extension):
""" Get mime-type and format by file extension.
This only accepts "txt".
"""
if extension.lower() == 'txt':
return 'text/plain', 'TXT'
elif extension.lower() == 'png':
return 'image/png', 'PNG'
else:
raise KnownUnknown('MirrorOSM only makes .txt and .png tiles, not "%s"' % extension)
def renderTile(self, width, height, srs, coord):
""" Render a single tile, return a SaveableResponse instance.
"""
##return EmptyResponse(None, coord)
tile = VectorTile(self.extents)
conn = _connect(self.dbdsn)
db = conn.cursor()
register_hstore(conn, True, False)
try:
db.execute(self.query_tile, (coord.column, coord.row, coord.zoom))
except Exception, e:
logging.error("db: %s\n %s", coord, e)
raise KnownUnknown("query failed")
def getTypeByExtension(self, extension):
""" Get mime-type and format by file extension.
This only accepts png (image), json (utf_grid interactivity), or geojson (vector)".
"""
if self.output_format == "utf_grid":
if extension.lower() != 'json':
raise KnownUnknown(
'FourSquare provider only makes .json tiles, not "%s"' %
extension)
if extension.lower() == 'json':
return 'text/json', 'JSON'
if extension.lower() == 'geojson':
return 'text/json', 'GeoJSON'
if extension.lower() == 'png':
return 'image/png', 'PNG'
raise KnownUnknown(
'FourSquare Provider only makes .geojson, .json, and .png tiles, not "%s"'
% extension)
def save(self, out, format):
if format not in ('PNG'):
raise KnownUnknown('We only saves and .png tiles, not "%s"' %
format)
im = Image.new('RGBA', (self.width, self.height))
dr = ImageDraw.Draw(im)
for data in self.grid:
if not data['has_stuff']:
continue
fill = hex_to_rgb(data['fill'])
x, y = data['rect'][0]
dr.rectangle(data['rect'], fill=fill)
im.save(out, 'PNG')
class Provider:
"""
"""
def __init__(self, layer, dsn, query_tile, query_poi):
logging.info("init %s", query_tile)
self.extension = 'vtm'
self.layer = layer
self.dbdsn = dsn
self.extents = 4096
self.query_tile = query_tile
self.query_tile_poi = query_poi
def renderTile(self, width, height, srs, coord):
""" Render a single tile, return a SaveableResponse instance.
"""
##return EmptyResponse(None, coord)
tile = VectorTile(self.extents)
conn = _connect(self.dbdsn)
db = conn.cursor()
register_hstore(conn, True, False)
try:
db.execute(self.query_tile, (coord.column, coord.row, coord.zoom))
except Exception, e:
logging.error("db: %s\n %s", coord, e)
raise KnownUnknown("query failed")
rows = db.fetchall()
try:
db.execute(self.query_tile_poi, (coord.column, coord.row, coord.zoom))
except Exception, e:
logging.error("db: %s\n %s", coord, e)
raise KnownUnknown("query failed")
def save(self, out, format):
#
# Serialize
#
if format in ('GeoJSON'):
#print "GeoJSON: ", self.content
content = self.content
#if 'wkt' in content['crs']:
# content['crs'] = {'type': 'link', 'properties': {'href': '0.wkt', 'type': 'ogcwkt'}}
#else:
# del content['crs']
elif format in ('PNG'):
content = self.content
else:
raise KnownUnknown(
'FourSquare response only saves .png, .json, and .geojson tiles, not "%s"'
% format)
#
# Encode
#
if format in ('GeoJSON'):
#indent = self.verbose and 2 or None
indent = 2
encoded = JSONEncoder(indent=indent).iterencode(content)
out.write(encoded)
elif format in ('JSON'):
out.write(content)
elif format in ('PNG'):
out.write(content)
# determine if we have enough information to prep a config and layer
has_fake_destination = bool(options.outputdirectory
or options.mbtiles_output)
has_fake_source = bool(options.mbtiles_input)
if has_fake_destination and has_fake_source:
config_dict, config_dirpath = dict(
layers={}), '' # parseConfig(options.config)
layer_dict = dict()
config_dict['cache'] = dict(name='test')
config_dict['layers'][options.layer or 'tiles-layer'] = layer_dict
elif options.config is None:
raise KnownUnknown(
'Missing required configuration (--config) parameter.')
elif options.layer is None:
raise KnownUnknown('Missing required layer (--layer) parameter.')
else:
config_dict, config_dirpath = parseConfig(options.config)
if options.layer not in config_dict['layers']:
raise KnownUnknown(
'"%s" is not a layer I know about. Here are some that I do know about: %s.'
% (options.layer, ', '.join(
sorted(config_dict['layers'].keys()))))
layer_dict = config_dict['layers'][options.layer]
layer_dict[
dest='config',
help='Path to configuration file.')
parser.add_option('-l',
'--layer',
dest='layer',
help='Layer name from configuration.')
pathinfo_pat = re.compile(r'^(?P<z>\d+)/(?P<x>\d+)/(?P<y>\d+)\.(?P<e>\w+)$')
if __name__ == '__main__':
options, paths = parser.parse_args()
try:
if options.config is None:
raise KnownUnknown(
'Missing required configuration (--config) parameter.')
if options.layer is None:
raise KnownUnknown('Missing required layer (--layer) parameter.')
config = parseConfig(options.config)
if options.layer not in config.layers:
raise KnownUnknown(
'"%s" is not a layer I know about. Here are some that I do know about: %s.'
% (options.layer, ', '.join(sorted(config.layers.keys()))))
layer = config.layers[options.layer]
coords = []
