def _merc_bbox(z, x, y):
extent = float(1 << z)
return BoundingBox(
MERCATOR_WORLD_SIZE * (x / extent - 0.5),
MERCATOR_WORLD_SIZE * (0.5 - (y + 1) / extent),
MERCATOR_WORLD_SIZE * ((x + 1) / extent - 0.5),
MERCATOR_WORLD_SIZE * (0.5 - y / extent))
def _parse_bbox(self, ns_deg, is_ns, ew_deg, is_ew, res):
bottom = int(ns_deg)
left = int(ew_deg)
if is_ns == 'S':
bottom = -bottom
if is_ew == 'W':
left = -left
b = BoundingBox(left, bottom, left + 30, bottom + 20)
return b
def __init__(self, parent, fname, lon, lat):
self.ftp_server = parent.ftp_server
self.base_path = parent.base_path
self.download_options = parent.download_options
self.base_dir = parent.base_dir
self.fname = fname
assert self.fname.endswith('.zip')
self.lon = lon
self.lat = lat
self.bbox = BoundingBox(self.lon, self.lat - 1, self.lon + 1, self.lat)
def test_group_dispatch_tiles(self):
regions = [
Region(BoundingBox(-124.56, 32.4, -114.15, 42.03), [8, 10])
]
t = terrarium.Terrarium(regions, [])
class Batch(object):
def __init__(self, queue, max_batch_len):
self.queue = queue
self.batch = []
def append(self, job):
self.batch.append(job)
def flush(self):
for job in self.batch:
self.queue.append(job)
self.batch = []
class Queue(object):
def __init__(self):
self.expected = set([
(8, 41, 99),
(8, 43, 98)
])
def start_batch(self, max_batch_len):
return Batch(self, max_batch_len)
def append(self, tile):
coord = (tile['z'], tile['x'], tile['y'])
if coord in self.expected:
self.expected.remove(coord)
def flush(self):
pass
class FakeLogger(object):
def info(self, msg):
print msg
def warning(self, msg):
print msg
logger = FakeLogger()
queue = Queue()
d = dispatcher.GroupingDispatcher(queue, 10, logger, 1000)
for tile in t.generate_tiles():
d.append(tile.freeze_dry())
d.flush()
self.assertEqual(queue.expected, set([]))
def test_generate_tiles(self):
regions = [
Region(BoundingBox(-124.56, 32.4, -114.15, 42.03), [8, 10])
]
t = terrarium.Terrarium(regions, [])
expected = set([
(8, 41, 99),
(8, 43, 98)
])
for tile in t.generate_tiles():
coord = (tile.z, tile.x, tile.y)
if coord in expected:
expected.remove(coord)
self.assertEqual(expected, set([]))
def _parse_bbox(self, link):
m = IS_SRTM_FILE.match(link)
if not m:
return None
is_ns, ns_deg, is_ew, ew_deg = m.groups()
bottom = int(ns_deg)
left = int(ew_deg)
if is_ns == 'S':
bottom = -bottom
if is_ew == 'W':
left = -left
return BoundingBox(left, bottom, left + 1, bottom + 1)
def test_dispatch_tiles(self):
regions = [
Region(BoundingBox(-124.56, 32.4, -114.15, 42.03), [8, 10])
]
t = terrarium.Terrarium(regions, [])
class Batch(object):
def __init__(self, queue, max_batch_len):
self.queue = queue
self.batch = []
def append(self, job):
self.batch.append(job)
def flush(self):
for job in self.batch:
self.queue.append(job)
self.batch = []
class Queue(object):
def __init__(self):
self.expected = set([
(8, 41, 99),
(8, 43, 98)
])
def start_batch(self, max_batch_len):
return Batch(self, max_batch_len)
def append(self, tile):
coord = (tile.z, tile.x, tile.y)
if coord in self.expected:
self.expected.remove(coord)
def flush(self):
pass
logger = logging.getLogger('process')
queue = Queue()
d = dispatcher.Dispatcher(queue, 10, logger)
for tile in t.generate_tiles():
d.append(tile)
d.flush()
self.assertEqual(queue.expected, set([]))
def _parse_ned_tile(fname, parent):
m = UNIVERSAL_NED_PATTERN.match(fname)
if not m:
return None
y = int(m.group(2)) + float(m.group(3)) / 100.0
x = int(m.group(5)) + float(m.group(6)) / 100.0
if m.group(1) == 's':
y = -y
if m.group(4) == 'w':
x = -x
bbox = BoundingBox(x, y - 0.25, x + 0.25, y)
state_code = m.group(7)
region_name = m.group(8)
year = int(m.group(9))
return NEDTile(parent, state_code, region_name, year, bbox)
def downloads_for(self, tile):
tiles = set()
# if the tile scale is greater than 20x the GMTED scale, then there's no
# point in including GMTED, it'll be far too fine to make a difference.
# GMTED is 7.5 arc seconds at best (30 at the poles).
if tile.max_resolution() > 20 * 7.5 / 3600:
return tiles
# buffer by 0.1 degrees (48px) to grab neighbouring tiles to ensure
# that there's no tile edge artefacts.
tile_bbox = tile.latlon_bbox().buffer(0.1)
for y in self.ys:
for x in self.xs:
bbox = BoundingBox(x, y, x + 30, y + 20)
if tile_bbox.intersects(bbox):
tiles.add(GMTEDTile(self, x, y))
return tiles
def rehydrate(self, data):
typ = 'ned_topobathy' if self.is_topobathy else 'ned'
assert data.get('type') == typ, \
"Unable to rehydrate %r in NED(%r)." % (data, typ)
return NEDTile(self, data['state_code'], data['region_name'],
data['year'], BoundingBox(*data['bbox']))
def latlon_bbox(self):
return BoundingBox(*self.bbox)
def _bbox(x, y):
return BoundingBox((x - 180) - HALF_ARC_SEC, (y - 90) - HALF_ARC_SEC,
(x - 179) + HALF_ARC_SEC, (y - 89) + HALF_ARC_SEC)
def latlon_bbox(self, z, x, y):
merc = _merc_bbox(z, x, y)
return BoundingBox(*_tx_bbox(self.tx, merc.bounds))
def test_intersection(self):
self.assertTrue(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(0, 0, 1, 1)))
self.assertTrue(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(1, 0, 2, 1)))
self.assertTrue(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(1, 1, 2, 2)))
self.assertTrue(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(0, 1, 1, 2)))
self.assertTrue(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(-1, 1, 0, 2)))
self.assertTrue(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(-1, 0, 0, 1)))
self.assertTrue(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(-1, -1, 0, 0)))
self.assertTrue(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(0, -1, 1, 0)))
self.assertTrue(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(1, -1, 2, 0)))
self.assertFalse(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(2, 2, 3, 3)))
self.assertFalse(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(0, 2, 1, 3)))
self.assertFalse(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(-2, 2, -1, 3)))
self.assertFalse(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(-2, 0, -1, 1)))
self.assertFalse(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(-2, -2, -1, -1)))
self.assertFalse(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(0, -2, 1, -1)))
self.assertFalse(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(2, -2, 3, -1)))
self.assertFalse(
BoundingBox(0, 0, 1, 1).intersects(BoundingBox(2, 0, 3, 1)))
def _merc_bbox(z, x, y):
extent = float(1 << z)
return BoundingBox(MERCATOR_WORLD_SIZE * (old_div(x, extent) - 0.5),
MERCATOR_WORLD_SIZE * (0.5 - old_div((y + 1), extent)),
MERCATOR_WORLD_SIZE * (old_div((x + 1), extent) - 0.5),
MERCATOR_WORLD_SIZE * (0.5 - old_div(y, extent)))
import logging
from osgeo import gdal
import tarfile
# hard-coded bounding boxes for each of the Great Lake datasets. this is because
# there's no link between the name of each piece of data and its bounding box,
# which was a design flaw / assumption for other data sources.
#
# also, we store the vertical offset of the dataset datum which doesn't appear
# to be part of the dataset itself. the vertical datums were obtained from
# https://tidesandcurrents.noaa.gov/gldatums.html
#
GREAT_LAKES = {
'erie': {
'bbox': BoundingBox(-84.0004167, 41.0004166, -78.0004166, 43.0004167),
'datum': 173.5
},
'huron': {
'bbox': BoundingBox(-84.5004167, 43.0004166, -79.6837500, 46.5004167),
'datum': 176.0
},
'michigan': {
'bbox': BoundingBox(-88.0004167, 41.6237499, -84.5004166, 46.0904167),
'datum': 176.0
},
'ontario': {
'bbox': BoundingBox(-79.9004167, 43.1504166, -76.0504166, 44.2504167),
'datum': 74.2
},
'superior': {
