def test_files(self, session):
today = util.utcnow().date()
rows = [
dict(time=today, lat=12.345, lon=12.345),
dict(time=today, lat=0, lon=12.345),
dict(time=today, lat=-10.000, lon=-11.000),
]
for row in rows:
lat, lon = DataMap.scale(row["lat"], row["lon"])
data = DataMap.shard_model(lat, lon)(
grid=(lat, lon), created=row["time"], modified=row["time"]
)
session.add(data)
session.flush()
lines = []
rows = 0
with util.selfdestruct_tempdir() as temp_dir:
quaddir = os.path.join(temp_dir, "quadtrees")
os.mkdir(quaddir)
shapes = os.path.join(temp_dir, "shapes")
tiles = os.path.join(temp_dir, "tiles")
for shard_id, shard in DataMap.shards().items():
filename = "map_%s.csv.gz" % shard_id
filepath = os.path.join(temp_dir, filename)
result = export_file(filepath, shard.__tablename__, _session=session)
if not result:
assert not os.path.isfile(filepath)
continue
rows += result
with util.gzip_open(filepath, "r") as fd:
written = fd.read()
lines.extend([line.split(",") for line in written.split()])
encode_file(filename, temp_dir, quaddir)
quadfolder = os.path.join(quaddir, "map_" + shard_id)
assert os.path.isdir(quadfolder)
self._check_quadtree(quadfolder)
merge_files(quaddir, shapes)
self._check_quadtree(shapes)
render_tiles(shapes, tiles, 1, 2)
assert sorted(os.listdir(tiles)) == ["0", "1", "2"]
assert sorted(os.listdir(os.path.join(tiles, "0", "0"))) == [
"0.png",
"*****@*****.**",
]
assert rows == 18
assert len(lines) == 18
lats = [round(float(line[0]), 2) for line in lines]
longs = [round(float(line[1]), 2) for line in lines]
assert set(lats) == set([-10.0, 0.0, 12.35])
assert set(longs) == set([-11.0, 12.35])
def test_files(self, db_rw, session):
today = util.utcnow().date()
rows = [
dict(time=today, lat=12.345, lon=12.345),
dict(time=today, lat=0, lon=12.345),
dict(time=today, lat=-10.000, lon=-11.000),
]
for row in rows:
lat, lon = DataMap.scale(row['lat'], row['lon'])
data = DataMap.shard_model(lat, lon)(
grid=(lat, lon), created=row['time'], modified=row['time'])
session.add(data)
session.flush()
lines = []
rows = 0
db_url = str(db_rw.engine.url)
with util.selfdestruct_tempdir() as temp_dir:
quaddir = os.path.join(temp_dir, 'quadtrees')
os.mkdir(quaddir)
shapes = os.path.join(temp_dir, 'shapes')
tiles = os.path.join(temp_dir, 'tiles')
for shard_id, shard in DataMap.shards().items():
filename = 'map_%s.csv.gz' % shard_id
filepath = os.path.join(temp_dir, filename)
result = export_file(
db_url, filepath, shard.__tablename__,
_session=session)
if not result:
assert not os.path.isfile(filepath)
continue
rows += result
with util.gzip_open(filepath, 'r') as fd:
written = fd.read()
lines.extend([line.split(',') for line in written.split()])
encode_file(filename, temp_dir, quaddir, DATAMAPS_DIR)
quadfolder = os.path.join(quaddir, 'map_' + shard_id)
assert os.path.isdir(quadfolder)
self._check_quadtree(quadfolder)
merge_files(quaddir, shapes, DATAMAPS_DIR)
self._check_quadtree(shapes)
render_tiles(shapes, tiles, 1, 2, DATAMAPS_DIR, PNGQUANT)
assert (sorted(os.listdir(tiles)) == ['0', '1', '2'])
assert (sorted(os.listdir(os.path.join(tiles, '0', '0'))) ==
['0.png', '*****@*****.**'])
assert rows == 36
assert len(lines) == 36
assert (set([round(float(l[0]), 2) for l in lines]) ==
set([-10.0, 0.0, 12.35]))
assert (set([round(float(l[1]), 2) for l in lines]) ==
set([-11.0, 12.35]))
def test_files(self, db, session): # pragma: no cover
today = util.utcnow().date()
rows = [
dict(time=today, lat=12.345, lon=12.345),
dict(time=today, lat=0, lon=12.345),
dict(time=today, lat=-10.000, lon=-11.000),
]
for row in rows:
lat, lon = DataMap.scale(row['lat'], row['lon'])
data = DataMap.shard_model(lat, lon)(grid=(lat, lon),
created=row['time'],
modified=row['time'])
session.add(data)
session.flush()
lines = []
rows = 0
with util.selfdestruct_tempdir() as temp_dir:
quaddir = os.path.join(temp_dir, 'quadtrees')
os.mkdir(quaddir)
shapes = os.path.join(temp_dir, 'shapes')
tiles = os.path.join(temp_dir, 'tiles')
for shard_id, shard in DataMap.shards().items():
filename = 'map_%s.csv.gz' % shard_id
filepath = os.path.join(temp_dir, filename)
result = export_file(filepath,
shard.__tablename__,
_session=session)
if not result:
assert not os.path.isfile(filepath)
continue
rows += result
with util.gzip_open(filepath, 'r') as fd:
written = fd.read()
lines.extend([line.split(',') for line in written.split()])
encode_file(filename, temp_dir, quaddir)
quadfolder = os.path.join(quaddir, 'map_' + shard_id)
assert os.path.isdir(quadfolder)
self._check_quadtree(quadfolder)
merge_files(quaddir, shapes)
self._check_quadtree(shapes)
render_tiles(shapes, tiles, 1, 2)
assert (sorted(os.listdir(tiles)) == ['0', '1', '2'])
assert (sorted(os.listdir(os.path.join(
tiles, '0', '0'))) == ['0.png', '*****@*****.**'])
assert rows == 18
assert len(lines) == 18
assert (set([round(float(l[0]), 2)
for l in lines]) == set([-10.0, 0.0, 12.35]))
assert (set([round(float(l[1]), 2)
for l in lines]) == set([-11.0, 12.35]))
def test_files(self, temp_dir, mock_db_worker_session):
lines = []
rows = 0
csvdir = os.path.join(temp_dir, "csv")
os.mkdir(csvdir)
quaddir = os.path.join(temp_dir, "quadtrees")
os.mkdir(quaddir)
shapes = os.path.join(temp_dir, "shapes")
tiles = os.path.join(temp_dir, "tiles")
expected = {"ne": (0, 0), "nw": (0, 0), "se": (12, 1), "sw": (6, 1)}
for shard_id, shard in DataMap.shards().items():
filename = f"map_{shard_id}.csv"
filepath = os.path.join(csvdir, filename)
row_count, file_count = export_to_csv(filename, csvdir,
shard.__tablename__)
assert row_count == expected[shard_id][0]
assert file_count == expected[shard_id][1]
if not row_count:
assert not os.path.isfile(filepath)
continue
rows += row_count
with open(filepath, "r") as fd:
written = fd.read()
lines.extend([line.split(",") for line in written.split()])
csv_to_quadtree(filename, csvdir, quaddir)
quadfolder = os.path.join(quaddir, "map_" + shard_id)
assert os.path.isdir(quadfolder)
self._check_quadtree(quadfolder)
assert rows
merge_quadtrees(quaddir, shapes)
self._check_quadtree(shapes)
with Pool() as pool:
render_tiles(pool, shapes, tiles, max_zoom=2)
assert sorted(os.listdir(tiles)) == ["0", "1", "2"]
assert sorted(os.listdir(os.path.join(tiles, "0", "0"))) == [
"0.png",
"*****@*****.**",
]
assert rows == 18
assert len(lines) == 18
lats = [round(float(line[0]), 2) for line in lines]
longs = [round(float(line[1]), 2) for line in lines]
assert set(lats) == set([-10.0, 0.0, 12.35])
assert set(longs) == set([-11.0, 12.35])
def test_files(self):
today = util.utcnow().date()
rows = [
dict(time=today, lat=12.345, lon=12.345),
dict(time=today, lat=0, lon=12.345),
dict(time=today, lat=-10.000, lon=-11.000),
]
for row in rows:
lat, lon = DataMap.scale(row["lat"], row["lon"])
data = DataMap.shard_model(lat, lon)(grid=(lat, lon), created=row["time"], modified=row["time"])
self.session.add(data)
self.session.flush()
lines = []
rows = 0
with util.selfdestruct_tempdir() as temp_dir:
quaddir = os.path.join(temp_dir, "quadtrees")
os.mkdir(quaddir)
shapes = os.path.join(temp_dir, "shapes")
tiles = os.path.join(temp_dir, "tiles")
for shard_id, shard in DATAMAP_SHARDS.items():
filename = "map_%s.csv.gz" % shard_id
filepath = os.path.join(temp_dir, filename)
result = export_file(None, filepath, shard.__tablename__, _db_rw=_make_db(), _session=self.session)
if not result:
self.assertFalse(os.path.isfile(filepath))
continue
rows += result
with util.gzip_open(filepath, "r") as fd:
written = fd.read()
lines.extend([line.split(",") for line in written.split()])
encode_file(filename, temp_dir, quaddir, DATAMAPS_DIR)
quadfolder = os.path.join(quaddir, "map_" + shard_id)
self.assertTrue(os.path.isdir(quadfolder))
self._check_quadtree(quadfolder)
merge_files(quaddir, shapes, DATAMAPS_DIR)
self._check_quadtree(shapes)
render_tiles(shapes, tiles, 1, 2, DATAMAPS_DIR, PNGQUANT)
self.assertEqual(sorted(os.listdir(tiles)), ["0", "1", "2"])
self.assertEqual(sorted(os.listdir(os.path.join(tiles, "0", "0"))), ["0.png", "*****@*****.**"])
self.assertEqual(rows, 36)
self.assertEqual(len(lines), 36)
self.assertEqual(set([round(float(l[0]), 2) for l in lines]), set([-10.0, 0.0, 12.35]))
self.assertEqual(set([round(float(l[1]), 2) for l in lines]), set([-11.0, 12.35]))