def test_files(self, session): today = util.utcnow().date() rows = [ dict(time=today, lat=12.345, lon=12.345), dict(time=today, lat=0, lon=12.345), dict(time=today, lat=-10.000, lon=-11.000), ] for row in rows: lat, lon = DataMap.scale(row["lat"], row["lon"]) data = DataMap.shard_model(lat, lon)( grid=(lat, lon), created=row["time"], modified=row["time"] ) session.add(data) session.flush() lines = [] rows = 0 with util.selfdestruct_tempdir() as temp_dir: quaddir = os.path.join(temp_dir, "quadtrees") os.mkdir(quaddir) shapes = os.path.join(temp_dir, "shapes") tiles = os.path.join(temp_dir, "tiles") for shard_id, shard in DataMap.shards().items(): filename = "map_%s.csv.gz" % shard_id filepath = os.path.join(temp_dir, filename) result = export_file(filepath, shard.__tablename__, _session=session) if not result: assert not os.path.isfile(filepath) continue rows += result with util.gzip_open(filepath, "r") as fd: written = fd.read() lines.extend([line.split(",") for line in written.split()]) encode_file(filename, temp_dir, quaddir) quadfolder = os.path.join(quaddir, "map_" + shard_id) assert os.path.isdir(quadfolder) self._check_quadtree(quadfolder) merge_files(quaddir, shapes) self._check_quadtree(shapes) render_tiles(shapes, tiles, 1, 2) assert sorted(os.listdir(tiles)) == ["0", "1", "2"] assert sorted(os.listdir(os.path.join(tiles, "0", "0"))) == [ "0.png", "*****@*****.**", ] assert rows == 18 assert len(lines) == 18 lats = [round(float(line[0]), 2) for line in lines] longs = [round(float(line[1]), 2) for line in lines] assert set(lats) == set([-10.0, 0.0, 12.35]) assert set(longs) == set([-11.0, 12.35])
def test_files(self, db_rw, session): today = util.utcnow().date() rows = [ dict(time=today, lat=12.345, lon=12.345), dict(time=today, lat=0, lon=12.345), dict(time=today, lat=-10.000, lon=-11.000), ] for row in rows: lat, lon = DataMap.scale(row['lat'], row['lon']) data = DataMap.shard_model(lat, lon)( grid=(lat, lon), created=row['time'], modified=row['time']) session.add(data) session.flush() lines = [] rows = 0 db_url = str(db_rw.engine.url) with util.selfdestruct_tempdir() as temp_dir: quaddir = os.path.join(temp_dir, 'quadtrees') os.mkdir(quaddir) shapes = os.path.join(temp_dir, 'shapes') tiles = os.path.join(temp_dir, 'tiles') for shard_id, shard in DataMap.shards().items(): filename = 'map_%s.csv.gz' % shard_id filepath = os.path.join(temp_dir, filename) result = export_file( db_url, filepath, shard.__tablename__, _session=session) if not result: assert not os.path.isfile(filepath) continue rows += result with util.gzip_open(filepath, 'r') as fd: written = fd.read() lines.extend([line.split(',') for line in written.split()]) encode_file(filename, temp_dir, quaddir, DATAMAPS_DIR) quadfolder = os.path.join(quaddir, 'map_' + shard_id) assert os.path.isdir(quadfolder) self._check_quadtree(quadfolder) merge_files(quaddir, shapes, DATAMAPS_DIR) self._check_quadtree(shapes) render_tiles(shapes, tiles, 1, 2, DATAMAPS_DIR, PNGQUANT) assert (sorted(os.listdir(tiles)) == ['0', '1', '2']) assert (sorted(os.listdir(os.path.join(tiles, '0', '0'))) == ['0.png', '*****@*****.**']) assert rows == 36 assert len(lines) == 36 assert (set([round(float(l[0]), 2) for l in lines]) == set([-10.0, 0.0, 12.35])) assert (set([round(float(l[1]), 2) for l in lines]) == set([-11.0, 12.35]))
def test_files(self, db, session): # pragma: no cover today = util.utcnow().date() rows = [ dict(time=today, lat=12.345, lon=12.345), dict(time=today, lat=0, lon=12.345), dict(time=today, lat=-10.000, lon=-11.000), ] for row in rows: lat, lon = DataMap.scale(row['lat'], row['lon']) data = DataMap.shard_model(lat, lon)(grid=(lat, lon), created=row['time'], modified=row['time']) session.add(data) session.flush() lines = [] rows = 0 with util.selfdestruct_tempdir() as temp_dir: quaddir = os.path.join(temp_dir, 'quadtrees') os.mkdir(quaddir) shapes = os.path.join(temp_dir, 'shapes') tiles = os.path.join(temp_dir, 'tiles') for shard_id, shard in DataMap.shards().items(): filename = 'map_%s.csv.gz' % shard_id filepath = os.path.join(temp_dir, filename) result = export_file(filepath, shard.__tablename__, _session=session) if not result: assert not os.path.isfile(filepath) continue rows += result with util.gzip_open(filepath, 'r') as fd: written = fd.read() lines.extend([line.split(',') for line in written.split()]) encode_file(filename, temp_dir, quaddir) quadfolder = os.path.join(quaddir, 'map_' + shard_id) assert os.path.isdir(quadfolder) self._check_quadtree(quadfolder) merge_files(quaddir, shapes) self._check_quadtree(shapes) render_tiles(shapes, tiles, 1, 2) assert (sorted(os.listdir(tiles)) == ['0', '1', '2']) assert (sorted(os.listdir(os.path.join( tiles, '0', '0'))) == ['0.png', '*****@*****.**']) assert rows == 18 assert len(lines) == 18 assert (set([round(float(l[0]), 2) for l in lines]) == set([-10.0, 0.0, 12.35])) assert (set([round(float(l[1]), 2) for l in lines]) == set([-11.0, 12.35]))
def test_files(self): today = util.utcnow().date() rows = [ dict(time=today, lat=12.345, lon=12.345), dict(time=today, lat=0, lon=12.345), dict(time=today, lat=-10.000, lon=-11.000), ] for row in rows: lat, lon = DataMap.scale(row["lat"], row["lon"]) data = DataMap.shard_model(lat, lon)(grid=(lat, lon), created=row["time"], modified=row["time"]) self.session.add(data) self.session.flush() lines = [] rows = 0 with util.selfdestruct_tempdir() as temp_dir: quaddir = os.path.join(temp_dir, "quadtrees") os.mkdir(quaddir) shapes = os.path.join(temp_dir, "shapes") tiles = os.path.join(temp_dir, "tiles") for shard_id, shard in DATAMAP_SHARDS.items(): filename = "map_%s.csv.gz" % shard_id filepath = os.path.join(temp_dir, filename) result = export_file(None, filepath, shard.__tablename__, _db_rw=_make_db(), _session=self.session) if not result: self.assertFalse(os.path.isfile(filepath)) continue rows += result with util.gzip_open(filepath, "r") as fd: written = fd.read() lines.extend([line.split(",") for line in written.split()]) encode_file(filename, temp_dir, quaddir, DATAMAPS_DIR) quadfolder = os.path.join(quaddir, "map_" + shard_id) self.assertTrue(os.path.isdir(quadfolder)) self._check_quadtree(quadfolder) merge_files(quaddir, shapes, DATAMAPS_DIR) self._check_quadtree(shapes) render_tiles(shapes, tiles, 1, 2, DATAMAPS_DIR, PNGQUANT) self.assertEqual(sorted(os.listdir(tiles)), ["0", "1", "2"]) self.assertEqual(sorted(os.listdir(os.path.join(tiles, "0", "0"))), ["0.png", "*****@*****.**"]) self.assertEqual(rows, 36) self.assertEqual(len(lines), 36) self.assertEqual(set([round(float(l[0]), 2) for l in lines]), set([-10.0, 0.0, 12.35])) self.assertEqual(set([round(float(l[1]), 2) for l in lines]), set([-11.0, 12.35]))