def test_files(self, session):
today = util.utcnow().date()
rows = [
dict(time=today, lat=12.345, lon=12.345),
dict(time=today, lat=0, lon=12.345),
dict(time=today, lat=-10.000, lon=-11.000),
]
for row in rows:
lat, lon = DataMap.scale(row["lat"], row["lon"])
data = DataMap.shard_model(lat, lon)(
grid=(lat, lon), created=row["time"], modified=row["time"]
)
session.add(data)
session.flush()
lines = []
rows = 0
with util.selfdestruct_tempdir() as temp_dir:
quaddir = os.path.join(temp_dir, "quadtrees")
os.mkdir(quaddir)
shapes = os.path.join(temp_dir, "shapes")
tiles = os.path.join(temp_dir, "tiles")
for shard_id, shard in DataMap.shards().items():
filename = "map_%s.csv.gz" % shard_id
filepath = os.path.join(temp_dir, filename)
result = export_file(filepath, shard.__tablename__, _session=session)
if not result:
assert not os.path.isfile(filepath)
continue
rows += result
with util.gzip_open(filepath, "r") as fd:
written = fd.read()
lines.extend([line.split(",") for line in written.split()])
encode_file(filename, temp_dir, quaddir)
quadfolder = os.path.join(quaddir, "map_" + shard_id)
assert os.path.isdir(quadfolder)
self._check_quadtree(quadfolder)
merge_files(quaddir, shapes)
self._check_quadtree(shapes)
render_tiles(shapes, tiles, 1, 2)
assert sorted(os.listdir(tiles)) == ["0", "1", "2"]
assert sorted(os.listdir(os.path.join(tiles, "0", "0"))) == [
"0.png",
"*****@*****.**",
]
assert rows == 18
assert len(lines) == 18
lats = [round(float(line[0]), 2) for line in lines]
longs = [round(float(line[1]), 2) for line in lines]
assert set(lats) == set([-10.0, 0.0, 12.35])
assert set(longs) == set([-11.0, 12.35])
def test_files(self, db_rw, session):
today = util.utcnow().date()
rows = [
dict(time=today, lat=12.345, lon=12.345),
dict(time=today, lat=0, lon=12.345),
dict(time=today, lat=-10.000, lon=-11.000),
]
for row in rows:
lat, lon = DataMap.scale(row['lat'], row['lon'])
data = DataMap.shard_model(lat, lon)(
grid=(lat, lon), created=row['time'], modified=row['time'])
session.add(data)
session.flush()
lines = []
rows = 0
db_url = str(db_rw.engine.url)
with util.selfdestruct_tempdir() as temp_dir:
quaddir = os.path.join(temp_dir, 'quadtrees')
os.mkdir(quaddir)
shapes = os.path.join(temp_dir, 'shapes')
tiles = os.path.join(temp_dir, 'tiles')
for shard_id, shard in DataMap.shards().items():
filename = 'map_%s.csv.gz' % shard_id
filepath = os.path.join(temp_dir, filename)
result = export_file(
db_url, filepath, shard.__tablename__,
_session=session)
if not result:
assert not os.path.isfile(filepath)
continue
rows += result
with util.gzip_open(filepath, 'r') as fd:
written = fd.read()
lines.extend([line.split(',') for line in written.split()])
encode_file(filename, temp_dir, quaddir, DATAMAPS_DIR)
quadfolder = os.path.join(quaddir, 'map_' + shard_id)
assert os.path.isdir(quadfolder)
self._check_quadtree(quadfolder)
merge_files(quaddir, shapes, DATAMAPS_DIR)
self._check_quadtree(shapes)
render_tiles(shapes, tiles, 1, 2, DATAMAPS_DIR, PNGQUANT)
assert (sorted(os.listdir(tiles)) == ['0', '1', '2'])
assert (sorted(os.listdir(os.path.join(tiles, '0', '0'))) ==
['0.png', '*****@*****.**'])
assert rows == 36
assert len(lines) == 36
assert (set([round(float(l[0]), 2) for l in lines]) ==
set([-10.0, 0.0, 12.35]))
assert (set([round(float(l[1]), 2) for l in lines]) ==
set([-11.0, 12.35]))
def test_files(self, db, session): # pragma: no cover
today = util.utcnow().date()
rows = [
dict(time=today, lat=12.345, lon=12.345),
dict(time=today, lat=0, lon=12.345),
dict(time=today, lat=-10.000, lon=-11.000),
]
for row in rows:
lat, lon = DataMap.scale(row['lat'], row['lon'])
data = DataMap.shard_model(lat, lon)(grid=(lat, lon),
created=row['time'],
modified=row['time'])
session.add(data)
session.flush()
lines = []
rows = 0
with util.selfdestruct_tempdir() as temp_dir:
quaddir = os.path.join(temp_dir, 'quadtrees')
os.mkdir(quaddir)
shapes = os.path.join(temp_dir, 'shapes')
tiles = os.path.join(temp_dir, 'tiles')
for shard_id, shard in DataMap.shards().items():
filename = 'map_%s.csv.gz' % shard_id
filepath = os.path.join(temp_dir, filename)
result = export_file(filepath,
shard.__tablename__,
_session=session)
if not result:
assert not os.path.isfile(filepath)
continue
rows += result
with util.gzip_open(filepath, 'r') as fd:
written = fd.read()
lines.extend([line.split(',') for line in written.split()])
encode_file(filename, temp_dir, quaddir)
quadfolder = os.path.join(quaddir, 'map_' + shard_id)
assert os.path.isdir(quadfolder)
self._check_quadtree(quadfolder)
merge_files(quaddir, shapes)
self._check_quadtree(shapes)
render_tiles(shapes, tiles, 1, 2)
assert (sorted(os.listdir(tiles)) == ['0', '1', '2'])
assert (sorted(os.listdir(os.path.join(
tiles, '0', '0'))) == ['0.png', '*****@*****.**'])
assert rows == 18
assert len(lines) == 18
assert (set([round(float(l[0]), 2)
for l in lines]) == set([-10.0, 0.0, 12.35]))
assert (set([round(float(l[1]), 2)
for l in lines]) == set([-11.0, 12.35]))
def _add(self, entries):
for lat, lon, time in entries:
lat, lon = DataMap.scale(lat, lon)
self.session.add(
DataMap.shard_model(lat, lon)(grid=(lat, lon),
created=time,
modified=time))
self.session.flush()
def test_grid_bytes(self):
lat = 12000
lon = 34000
grid = encode_datamap_grid(lat, lon)
model = DataMap.shard_model(lat, lon)
self.session.add(model(grid=grid))
self.session.flush()
result = self.session.query(model).first()
self.assertEqual(result.grid, (lat, lon))
def test_fields(self):
today = util.utcnow().date()
lat = 12345
lon = -23456
model = DataMap.shard_model(lat, lon)
self.session.add(model(grid=(lat, lon), created=today, modified=today))
self.session.flush()
result = self.session.query(model).first()
self.assertEqual(result.grid, (lat, lon))
self.assertEqual(result.created, today)
self.assertEqual(result.modified, today)
def test_files(self):
today = util.utcnow().date()
rows = [
dict(time=today, lat=12.345, lon=12.345),
dict(time=today, lat=0, lon=12.345),
dict(time=today, lat=-10.000, lon=-11.000),
]
for row in rows:
lat, lon = DataMap.scale(row["lat"], row["lon"])
data = DataMap.shard_model(lat, lon)(grid=(lat, lon), created=row["time"], modified=row["time"])
self.session.add(data)
self.session.flush()
lines = []
rows = 0
with util.selfdestruct_tempdir() as temp_dir:
quaddir = os.path.join(temp_dir, "quadtrees")
os.mkdir(quaddir)
shapes = os.path.join(temp_dir, "shapes")
tiles = os.path.join(temp_dir, "tiles")
for shard_id, shard in DATAMAP_SHARDS.items():
filename = "map_%s.csv.gz" % shard_id
filepath = os.path.join(temp_dir, filename)
result = export_file(None, filepath, shard.__tablename__, _db_rw=_make_db(), _session=self.session)
if not result:
self.assertFalse(os.path.isfile(filepath))
continue
rows += result
with util.gzip_open(filepath, "r") as fd:
written = fd.read()
lines.extend([line.split(",") for line in written.split()])
encode_file(filename, temp_dir, quaddir, DATAMAPS_DIR)
quadfolder = os.path.join(quaddir, "map_" + shard_id)
self.assertTrue(os.path.isdir(quadfolder))
self._check_quadtree(quadfolder)
merge_files(quaddir, shapes, DATAMAPS_DIR)
self._check_quadtree(shapes)
render_tiles(shapes, tiles, 1, 2, DATAMAPS_DIR, PNGQUANT)
self.assertEqual(sorted(os.listdir(tiles)), ["0", "1", "2"])
self.assertEqual(sorted(os.listdir(os.path.join(tiles, "0", "0"))), ["0.png", "*****@*****.**"])
self.assertEqual(rows, 36)
self.assertEqual(len(lines), 36)
self.assertEqual(set([round(float(l[0]), 2) for l in lines]), set([-10.0, 0.0, 12.35]))
self.assertEqual(set([round(float(l[1]), 2) for l in lines]), set([-11.0, 12.35]))
def test_grid_list(self):
lat = 1000
lon = -2000
self.session.add(DataMap.shard_model(lat, lon)(grid=[lat, lon]))
with self.assertRaises(Exception):
self.session.flush()
def test_grid_length(self):
self.session.add(DataMap.shard_model(0, 9)(grid=b'\x00' * 9))
with self.assertRaises(Exception):
self.session.flush()
def test_grid_none(self):
self.session.add(DataMap.shard_model(0, 0)(grid=None))
with self.assertRaises(Exception):
self.session.flush()
def test_grid_length(self, session):
session.add(DataMap.shard_model(0, 9)(grid=b'\x00' * 9))
with pytest.raises(Exception):
session.flush()
def test_grid_none(self, session):
with warnings.catch_warnings():
warnings.simplefilter('ignore', SAWarning)
session.add(DataMap.shard_model(0, 0)(grid=None))
with pytest.raises(Exception):
session.flush()
def _add(self, entries):
for lat, lon, time in entries:
lat, lon = DataMap.scale(lat, lon)
self.session.add(DataMap.shard_model(lat, lon)(
grid=(lat, lon), created=time, modified=time))
self.session.flush()
def _one(self, lat, lon, time):
lat, lon = DataMap.scale(lat, lon)
return DataMap.shard_model(lat, lon)(grid=(lat, lon),
created=time,
modified=time)
def _one(self, lat, lon, time):
lat, lon = DataMap.scale(lat, lon)
return DataMap.shard_model(lat, lon)(
grid=(lat, lon), created=time, modified=time)
def test_grid_none(self, session):
session.add(DataMap.shard_model(0, 0)(grid=None))
with pytest.raises(Exception):
session.flush()
