def test_tilequeue_intersect_enqueues_coords(self):
from mock import MagicMock
from tilequeue.command import tilequeue_intersect
from ModestMaps.Core import Coordinate
from tilequeue.tile import serialize_coord
from tilequeue.tile import coord_marshall_int
cfg_mock = MagicMock()
cfg_mock.queue_type = 'sqs'
periperals_mock = MagicMock()
c0 = Coordinate(row=0, column=0, zoom=0)
c1 = Coordinate(row=1, column=1, zoom=1)
coords = (c0, c1)
periperals_mock.redis_cache_index = MagicMock(
fetch_tiles_of_interest=lambda: set(
map(coord_marshall_int, coords)))
queue_mock = MagicMock()
periperals_mock.queue = queue_mock
queue_mock.enqueue = self.fake_enqueue
queue_mock.enqueue_batch = self.fake_enqueue_batch
import os
with tempdir() as expired_tiles_location:
expected_file = os.path.join(expired_tiles_location,
'expire_list.txt')
with open(expected_file, "w+") as fp:
fp.write('\n'.join(map(serialize_coord, coords)))
cfg_mock.intersect_expired_tiles_location = expired_tiles_location
cfg_mock.logconfig = None
tilequeue_intersect(cfg_mock, periperals_mock)
self.assertIn(c0, self.enqueued_list)
self.assertIn(c1, self.enqueued_list)
def test_make_metatiles_multiple_coordinates(self):
# we need to be able to handle this so that we can do "cut out"
# overzoomed tiles at z>16.
json = "{\"json\":true}"
tiles = [
dict(tile=json, coord=Coordinate(17, 123, 456),
format=json_format, layer='all'),
dict(tile=json, coord=Coordinate(17, 123, 457),
format=json_format, layer='all'),
]
metatiles = make_metatiles(1, tiles)
self.assertEqual(2, len(metatiles))
coords = set([Coordinate(17, 123, 456), Coordinate(17, 123, 457)])
for meta in metatiles:
self.assertTrue(meta['coord'] in coords)
coords.remove(meta['coord'])
self.assertEqual('all', meta['layer'])
self.assertEqual(zip_format, meta['format'])
buf = StringIO.StringIO(meta['tile'])
with zipfile.ZipFile(buf, mode='r') as z:
self.assertEqual(json, z.open('0/0/0.json').read())
# check all coords were consumed
self.assertEqual(0, len(coords))
def test_tilequeue_intersect_does_not_enqueue_coords(self):
from mock import MagicMock
from tilequeue.command import tilequeue_intersect
from ModestMaps.Core import Coordinate
from tilequeue.tile import serialize_coord
cfg_mock = MagicMock()
cfg_mock.queue_type = 'sqs'
periperals_mock = MagicMock()
c0 = Coordinate(row=0, column=0, zoom=0)
c1 = Coordinate(row=1, column=1, zoom=1)
periperals_mock.redis_cache_index = MagicMock(
intersect=lambda x, y: ([]))
queue_mock = MagicMock()
periperals_mock.queue = queue_mock
queue_mock.enqueue = self.fake_enqueue
queue_mock.enqueue_batch = self.fake_enqueue_batch
import os
with tempdir() as expired_tiles_location:
expected_file = os.path.join(expired_tiles_location,
'expire_list.txt')
with open(expected_file, "w+") as fp:
fp.write(serialize_coord(c0) + "\n" + serialize_coord(c1))
cfg_mock.intersect_expired_tiles_location = expired_tiles_location
cfg_mock.logconfig = None
tilequeue_intersect(cfg_mock, periperals_mock)
self.assertNotIn(c0, self.enqueued_list)
self.assertNotIn(c1, self.enqueued_list)
def generateSubquads(row, column, zoom):
row0, col0, row1, col1, zoom1 \
= row*2, column*2, row*2+1, column*2+1, zoom+1
count = 4
offset = 0
for r in range(2):
for c in range(2):
if r == 0 and c == 0:
coord = Coordinate(row0, col0, zoom1)
if r == 0 and c == 1:
coord = Coordinate(row0, col1, zoom1)
if r == 1 and c == 0:
coord = Coordinate(row1, col0, zoom1)
if r == 1 and c == 1:
coord = Coordinate(row1, col1, zoom1)
#print coord
# Ensure we only yield coords in the area of interest
#if extentContainsCoord( render_bbox, coord ):
#print '\t', coord
yield (offset, count, coord)
#else:
# continue
offset += 1
def bounds_to_coords(bounds, zoom):
minx, miny, maxx, maxy = bounds
topleft_lng = minx
topleft_lat = maxy
bottomright_lat = miny
bottomright_lng = maxx
topleftx, toplefty = deg2num(topleft_lat, topleft_lng, zoom)
bottomrightx, bottomrighty = deg2num(bottomright_lat, bottomright_lng,
zoom)
# clamp max values
maxval = int(math.pow(2, zoom) - 1)
bottomrightx = min(maxval, bottomrightx)
bottomrighty = min(maxval, bottomrighty)
topleftcoord = Coordinate(row=toplefty, column=topleftx, zoom=zoom)
# check if one coordinate subsumes the whole bounds at this zoom
if topleftx == bottomrightx and toplefty == bottomrighty:
return [topleftcoord]
# we have two inclusive coordinates representing the range
bottomrightcoord = Coordinate(row=bottomrighty,
column=bottomrightx,
zoom=zoom)
return topleftcoord, bottomrightcoord
def test_2x2_tile_nominal_1(self):
from tilequeue.process import metatile_children_with_size
coord = Coordinate(zoom=0, column=0, row=0)
result = metatile_children_with_size(coord, 1, 0, 256)
self.assertEqual(set([
Coordinate(zoom=0, column=0, row=0),
]), set(result))
def iterate_squares(ds, zoom):
'''
'''
xoff, xstride, _, yoff, _, ystride = ds.GetGeoTransform()
minlon, maxlat = xoff, yoff
maxlon = xoff + ds.RasterXSize * xstride
minlat = yoff + ds.RasterYSize * ystride
if zoom > 11:
maxlat = min(58, maxlat)
osm = Provider()
ul = osm.locationCoordinate(Location(maxlat, minlon)).zoomTo(zoom)
lr = osm.locationCoordinate(Location(minlat, maxlon)).zoomTo(zoom)
#lr = osm.locationCoordinate(Location(20, -60)).zoomTo(zoom)
row = int(ul.row)
while row < lr.row:
lat = osm.coordinateLocation(Coordinate(row, 0, zoom)).lat
print >> sys.stderr, 'lat:', round(lat, 2)
col = int(ul.column)
while col < lr.column:
coord = Coordinate(row, col, zoom)
sw = osm.coordinateLocation(coord.down())
ne = osm.coordinateLocation(coord.right())
west = max(minlon, sw.lon)
north = min(maxlat, ne.lat)
east = min(maxlon, ne.lon)
south = max(minlat, sw.lat)
left = round((west - xoff) / xstride)
top = round((north - yoff) / ystride)
width = round((east - xoff) / xstride) - left
height = round((south - yoff) / ystride) - top
yield (coord, south, north, int(left), int(top), int(width), int(height))
col += 1
row += 1
return
x = xmin
while x < xmax:
print >> sys.stderr, 'lon:', x
y = ymin
while y < ymax:
left = round((x - xoff) / xstride)
top = round((y + size - yoff) / ystride)
width, height = round(size / xstride), round(size / -ystride)
yield (round(x, 2), round(y, 2), int(left), int(top), int(width), int(height))
y += size
x += size
def test_tiles_children(self):
from tilequeue.tile import coord_children
from ModestMaps.Core import Coordinate
coord = Coordinate(0, 0, 0)
children = coord_children(coord)
self.assertEqual(4, len(children))
self.assertEqual(Coordinate(0, 0, 1), children[0])
self.assertEqual(Coordinate(1, 0, 1), children[1])
self.assertEqual(Coordinate(0, 1, 1), children[2])
self.assertEqual(Coordinate(1, 1, 1), children[3])
def test_zoom_1(self):
tiles = self._call_fut(1)
self.assertEqual(5, len(tiles))
expected_tiles = [
Coordinate(0, 0, 0),
Coordinate(0, 0, 1),
Coordinate(1, 0, 1),
Coordinate(0, 1, 1),
Coordinate(1, 1, 1),
]
self._assert_tilelist(expected_tiles, tiles)
def test_enqueue_batch_does_not_add_redundant_tile_in_flight(self):
coords = [
Coordinate(row=1, column=1, zoom=1),
Coordinate(row=2, column=2, zoom=2)
]
mock = MagicMock()
mock.side_effect = [True, False]
self.mockRedis.sismember = mock
self.sqs.enqueue_batch(coords)
self.assertEqual(1, len(self.message_tuples))
self.assertEqual(self.message_tuples[0][1], "2/2/2")
def test_make_metatiles_single(self):
json = "{\"json\":true}"
tiles = [dict(tile=json, coord=Coordinate(0, 0, 0),
format=json_format, layer='all')]
metatiles = make_metatiles(1, tiles)
self.assertEqual(1, len(metatiles))
self.assertEqual(Coordinate(0, 0, 0), metatiles[0]['coord'])
self.assertEqual('all', metatiles[0]['layer'])
self.assertEqual(zip_format, metatiles[0]['format'])
buf = StringIO.StringIO(metatiles[0]['tile'])
with zipfile.ZipFile(buf, mode='r') as z:
self.assertEqual(json, z.open('0/0/0.json').read())
def test_enqueue_batch_adds_tiles(self):
coords = [
Coordinate(row=1, column=1, zoom=1),
Coordinate(row=2, column=2, zoom=2)
]
mock = MagicMock()
mock.side_effect = [False, False]
self.mockRedis.sismember = mock
self.sqs.enqueue_batch(coords)
self.assertEqual(2, len(self.message_tuples))
self.assertEqual(self.message_tuples[0][1], "1/1/1")
self.assertEqual(self.message_tuples[1][1], "2/2/2")
def test_enqueue_batch_adds_tiles_as_in_flight(self):
from tilequeue.tile import coord_marshall_int
coords = [
Coordinate(row=1, column=1, zoom=1),
Coordinate(row=2, column=2, zoom=2)
]
mock = MagicMock()
mock.side_effect = [False, False]
self.mockRedis.sismember = mock
self.mockRedis.sadd = self.fake_sadd
self.sqs.enqueue_batch(coords)
self.assertEqual(self.key_name, self.sqs.inflight_key)
exp_values = map(coord_marshall_int, coords)
self.assertEqual(exp_values, self.values)
def bbox_polygon(bbox, provider, zoom):
rectangle = bbox.envelope.exterior
(x1, y1), (x2, y2) = rectangle.coords[0], rectangle.coords[2]
coord1 = Coordinate(y1, x1, zoom + 8)
coord2 = Coordinate(y2, x2, zoom + 8)
location1 = provider.coordinateLocation(coord1)
location2 = provider.coordinateLocation(coord2)
lat1, lon1 = location1.lat, location1.lon
lat2, lon2 = location2.lat, location2.lon
return Polygon(((lon1, lat1), (lon1, lat2), (lon2, lat2), (lon2, lat1), (lon1, lat1)))
def test_roundtrip_serialization(self):
from tilequeue.tile import coord_marshall_int
from tilequeue.tile import coord_unmarshall_int
from tilequeue.tile import seed_tiles
from ModestMaps.Core import Coordinate
from itertools import chain
seed_coords = seed_tiles(0, 5)
example_coords = [
Coordinate(zoom=20, column=1002463, row=312816),
Coordinate(zoom=30, column=12345678, row=12345678),
]
coords = chain(seed_coords, example_coords)
for coord in coords:
self.assertEquals(coord,
coord_unmarshall_int(coord_marshall_int(coord)))
def fit_in_tile(z, x, y, shape):
"""
Fit shape into the tile. Shape should be a Shapely geometry or WKT string
with coordinates between 0 and 1. This unit square is then remapped into
the tile z/x/y.
"""
from ModestMaps.Core import Coordinate
from shapely.ops import transform
from shapely.wkt import loads as wkt_loads
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import reproject_mercator_to_lnglat
bounds = coord_to_mercator_bounds(Coordinate(zoom=z, column=x, row=y))
if isinstance(shape, (str, unicode)):
shape = wkt_loads(shape)
# check shape fits within unit square, so we can transform it to fit
# within the tile.
assert shape.bounds[0] >= 0
assert shape.bounds[1] >= 0
assert shape.bounds[2] <= 1
assert shape.bounds[3] <= 1
def _transform(x, y, *unused_coords):
return (
x * (bounds[2] - bounds[0]) + bounds[0],
y * (bounds[3] - bounds[1]) + bounds[1],
)
merc_shape = transform(_transform, shape)
return transform(reproject_mercator_to_lnglat, merc_shape)
def get(self, kernel_id, layer_name, x, y, z, extension, **kwargs):
config = self.ktile_config_manager[kernel_id]
layer = config.layers[layer_name]
coord = Coordinate(int(y), int(x), int(z))
# To run synchronously:
# status_code, headers, content = layer.getTileResponse(
# coord, extension)
status_code, headers, content = yield self.client.getTileResponse(
layer, coord, extension)
if layer.max_cache_age is not None:
expires = datetime.utcnow() + timedelta(
seconds=layer.max_cache_age)
headers['Expires'] = expires.strftime('%a %d %b %Y %H:%M:%S GMT')
headers['Cache-Control'] = 'public, max-age=%d' \
% layer.max_cache_age
else:
headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
headers['Pragma'] = 'no-cache'
headers['Expires'] = '0'
# Force allow cross origin access
headers["Access-Control-Allow-Origin"] = "*"
# Fill tornado handler properties with ktile code/header/content
for k, v in headers.items():
self.set_header(k, v)
self.set_status(status_code)
self.write(content)
def doRequest(self):
'''
http://localhost:8000/tms/test-client/qdjango/10/rt/15/17410/11915.png
http://localhost:8000/tms/test-client/qdjango/10/rt/13/4348/2979.png
:return:
'''
configDict = settings.TILESTACHE_CONFIG_BASE
configDict['layers'][self.layer_name] = Layer.objects.get(project_id=self.projectId, name=self.layer_name).tilestache_conf
'''
configDict['layers']['rt'] = {
"provider": {
"name": "url template",
"template": "http://www502.regione.toscana.it/wmsraster/com.rt.wms.RTmap/wms?map=wmspiapae&SERVICE=WMS&REQUEST=GetMap&VERSION=1.3.0&LAYERS=rt_piapae.carta_dei_caratteri_del_paesaggio.50k.ct.rt&STYLES=&FORMAT=image/png&TRANSPARENT=undefined&CRS=EPSG:3857&WIDTH=$width&HEIGHT=$height&bbox=$xmin,$ymin,$xmax,$ymax"
},
"projection": "spherical mercator"
}
'''
config = Config.buildConfiguration(configDict)
layer = config.layers[self.layer_name]
coord = Coordinate(int(self.tile_row), int(self.tile_column), int(self.tile_zoom))
tile_mimetype, tile_content = getTile(layer, coord, self.tile_format, ignore_cached=False)
return HttpResponse(content_type=tile_mimetype, content=tile_content)
def test_metatile_file_timing(self):
from time import gmtime, time
from tilequeue.metatile import metatiles_are_equal
# tilequeue's "GET before PUT" optimisation relies on being able to
# fetch a tile from S3 and compare it to the one that was just
# generated. to do this, we should try to make the tiles as similar
# as possible across multiple runs.
json = "{\"json\":true}"
tiles = [
dict(tile=json,
coord=Coordinate(0, 0, 0),
format=json_format,
layer='all')
]
when_will_then_be_now = 10
t = time()
now = gmtime(t)[0:6]
then = gmtime(t - when_will_then_be_now)[0:6]
metatile_1 = make_metatiles(1, tiles, then)
metatile_2 = make_metatiles(1, tiles, now)
self.assertTrue(
metatiles_are_equal(metatile_1[0]['tile'], metatile_2[0]['tile']))
def test_road_no_repeated_points(self):
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_bounds
z, x, y = 16, 17885, 27755
coord = Coordinate(zoom=z, column=x, row=y)
bounds = coord_to_bounds(coord)
# have to reorder the bounds from conventional order to the unusual
# scheme that overpass expects (south,west,north,east).
bbox = "%f,%f,%f,%f" % (bounds[1], bounds[0], bounds[3], bounds[2])
overpass = "******"
query = "way(" + bbox + ")[highway]%3B>%3B"
self.load_fixtures([overpass + query])
num_tested = 0
with self.features_in_tile_layer(z, x, y, 'roads') as features:
for feature in features:
gtype = feature['geometry']['type']
if gtype == 'LineString':
self._assert_no_repeated_points(
feature['geometry']['coordinates'])
num_tested += 1
elif gtype == 'MultiLineString':
for linestring in feature['geometry']['coordinates']:
self._assert_no_repeated_points(linestring)
num_tested += 1
self.assertTrue(num_tested != 0,
"Expected at least one testable feature.")
def test_successful_merge(self):
from ModestMaps.Core import Coordinate
from shapely.geometry import LineString
from tilequeue.tile import coord_to_bounds
import dsl
z, x, y = 9, 145, 201
bounds = coord_to_bounds(Coordinate(zoom=z, column=x, row=y))
mid_x = (bounds[2] - bounds[0]) / 2.0 + bounds[0]
mid_y = (bounds[3] - bounds[1]) / 2.0 + bounds[1]
ls1 = LineString([(mid_x-0.01, mid_y-0.01), (mid_x, mid_y)])
ls2 = LineString([(mid_x, mid_y), (mid_x+0.01, mid_y+0.01)])
props = dict(waterway=u'river', name=u'foo')
self.generate_fixtures(
dsl.way(1, ls1, props),
dsl.way(2, ls2, props),
)
self.assert_n_matching_features(
z, x, y, 'water', {
'name': 'foo',
'kind': 'river',
'label_placement': type(None),
}, 1)
with self.features_in_tile_layer(z, x, y, 'water') as features:
for f in features:
if 'label_placement' in f['properties']:
continue
assert f['geometry']['type'] == 'LineString'
assert len(f['geometry']['coordinates']) == 2
def way_full(way_id, zoom=16):
url = 'https://api.openstreetmap.org/api/0.6/way/%d/full' % (way_id,)
headers = {'user-agent': 'mktest.py/0.0.1 (https://github.com/tilezen)'}
r = requests.get(url, headers=headers)
root = ET.fromstring(r.content)
assert root.tag == 'osm'
nodes = {}
nds = []
tags = {}
for child in root:
if child.tag == 'node':
node_id = int(child.attrib['id'])
nodes[node_id] = child
elif child.tag == 'way':
assert way_id == int(child.attrib['id'])
for wchild in child:
if wchild.tag == 'nd':
nds.append(int(wchild.attrib['ref']))
elif wchild.tag == 'tag':
k = wchild.attrib['k']
v = wchild.attrib['v']
tags[k] = v
assert nds
first_node = nodes[nds[0]]
lat = float(first_node.attrib['lat'])
lon = float(first_node.attrib['lon'])
x, y = tile.deg2num(lat, lon, zoom)
tags['source'] = 'openstreetmap.org'
return Coordinate(zoom=zoom, column=x, row=y), tags
def node_test(args):
import json
position, node_tags = node(args.node_id, args.zoom)
x, y = tile.deg2num(position[1], position[0], args.zoom)
coord = Coordinate(zoom=args.zoom, column=x, row=y)
expect = json.loads(args.expect) if args.expect else None
if expect:
name = '_'.join(_make_ident(v) for v in expect.values())
else:
name = 'FIXME'
args = dict(
name=name,
z=args.zoom,
x=coord.column,
y=coord.row,
position=position,
node_id=args.node_id,
node_tags=node_tags,
expect=expect,
)
output = _render_template('node_test', args)
print output.encode('utf-8')
def coord_unmarshall_int(coord_int):
if isinstance(coord_int, (str, unicode)):
coord_int = int(coord_int)
zoom = zoom_mask & coord_int
row = row_mask & (coord_int >> row_offset)
column = col_mask & (coord_int >> col_offset)
return Coordinate(column=column, row=row, zoom=zoom)
def parse_request_path(path, extensions_to_handle):
"""given a path, parse the underlying layer, coordinate, and format"""
parts = path.split('/')
if parts[1] == "changeset-id":
return enqueue_expired_changeset(parts[2])
if len(parts) != 5:
return None
_, layer_spec, zoom_str, column_str, row_and_ext = parts
row_fields = row_and_ext.split('.')
if len(row_fields) != 2:
return None
row_str, ext = row_fields
if ext not in extensions_to_handle:
return None
format = extension_to_format.get(ext)
assert format, 'Unknown extension %s' % ext
try:
zoom = int(zoom_str)
column = int(column_str)
row = int(row_str)
except ValueError:
return None
coord = Coordinate(zoom=zoom, column=column, row=row)
if not coord_is_valid(coord):
return None
request_data = RequestData(layer_spec, coord, format)
return request_data
def _calc(metatile_size, tile_sizes):
coord = Coordinate(zoom=0, row=0, column=0)
metatile_zoom = metatile_zoom_from_size(metatile_size)
max_zoom = 16 - metatile_zoom
return calculate_sizes_by_zoom(
coord, metatile_zoom, tile_sizes, max_zoom)
def generateCoordinates(ul, lr, zooms, padding):
""" Generate a stream of (offset, count, coordinate) tuples for seeding.
Flood-fill coordinates based on two corners, a list of zooms and padding.
"""
# start with a simple total of all the coordinates we will need.
count = 0
for zoom in zooms:
ul_ = ul.zoomTo(zoom).container().left(padding).up(padding)
lr_ = lr.zoomTo(zoom).container().right(padding).down(padding)
rows = lr_.row + 1 - ul_.row
cols = lr_.column + 1 - ul_.column
count += int(rows * cols)
# now generate the actual coordinates.
# offset starts at zero
offset = 0
for zoom in zooms:
ul_ = ul.zoomTo(zoom).container().left(padding).up(padding)
lr_ = lr.zoomTo(zoom).container().right(padding).down(padding)
range_ = xrange if PY2 else range
for row in range_(int(ul_.row), int(lr_.row + 1)):
for column in range_(int(ul_.column), int(lr_.column + 1)):
coord = Coordinate(row, column, zoom)
yield (offset, count, coord)
offset += 1
def _overpass_element(layer_name, query_fn, args):
import json
result = query_fn(args.query)
pos = result.position
x, y = tile.deg2num(pos[0], pos[1], args.zoom)
coord = Coordinate(zoom=args.zoom, column=x, row=y)
expect = json.loads(args.expect) if args.expect else None
if expect:
name = '_'.join(_make_ident(v) for v in expect.values())
else:
name = 'FIXME'
name = name + '_' + result.element_type()
tags = result.tags
tags['source'] = 'openstreetmap.org'
params = dict(
name=name,
z=args.zoom,
x=coord.column,
y=coord.row,
elt_id=result.element_id,
tags=tags,
expect=expect,
layer_name=layer_name,
geom_fn_name=result.geom_fn_name(),
geom_fn_arg=result.geom_fn_arg(),
elt_type=result.element_type(),
)
output = _render_template('overpass_test', params)
print output.encode('utf-8')
def request(config_content, layer_name, format, row, column, zoom):
'''
Helper method to write config_file to disk and do
request
'''
if sys.version_info.major == 2:
is_string = isinstance(config_content, str)
else:
is_string = isinstance(config_content, (str, bytes))
if is_string:
absolute_file_name = create_temp_file(config_content)
config = parseConfig(absolute_file_name)
else:
config = parseConfig(config_content)
layer = config.layers[layer_name]
coord = Coordinate(int(row), int(column), int(zoom))
mime_type, tile_content = getTile(layer, coord, format)
if is_string:
os.remove(absolute_file_name)
return mime_type, tile_content
def tile(key, x, y, z):
dbo = resolve_dbo(key)
if key is None:
return abort(404)
start = time()
# TODO Add z limit -> 204 No Content
c = Coordinate(y, x, z)
nw = OSM.coordinateLocation(c)
se = OSM.coordinateLocation(c.down().right())
box = as_bbox(se, nw, 4326)
query = dbo.Feature.query.filter(Feature.geom.intersects(box))
features = []
for f in query:
feature = {
'type': 'Feature',
'id': f.id,
'properties': f.properties,
'geometry': f.shape.__geo_interface__
}
features.append(feature)
data_time = time() - start
response = jsonify({'type': 'FeatureCollection', 'features': features})
serialize_time = time() - start - data_time
# print (x,y,z), "Data:", data_time, "Serialize:", serialize_time
return response
