def update(self, batch=1000):
queue = self.task.app.data_queues['update_mapstat']
today = util.utcnow().date()
positions = queue.dequeue(batch=batch)
if not positions:
return 0
found = set()
wanted = set()
for position in positions:
wanted.add(MapStat.to_hashkey(lat=MapStat.scale(position['lat']),
lon=MapStat.scale(position['lon'])))
stat_iter = MapStat.iterkeys(
self.session, list(wanted),
extra=lambda query: query.options(load_only('lat', 'lon')))
found = set([stat.hashkey() for stat in stat_iter])
for key in (wanted - found):
stmt = MapStat.__table__.insert(
on_duplicate='id = id').values(
time=today, lat=key.lat, lon=key.lon)
self.session.execute(stmt)
if queue.size() >= batch:
self.task.apply_async(
kwargs={'batch': batch},
countdown=2,
expires=10)
return len(positions)
def __call__(self, batch=1000):
queue = self.task.app.data_queues['update_mapstat']
today = util.utcnow().date()
positions = queue.dequeue(batch=batch)
if not positions:
return 0
scaled_positions = set()
for position in positions:
scaled_positions.add(
MapStat.scale(position['lat'], position['lon']))
wanted = set()
for scaled in scaled_positions:
wanted.add(MapStat.to_hashkey(lat=scaled[0], lon=scaled[1]))
stat_iter = MapStat.iterkeys(
self.session, list(wanted),
extra=lambda query: query.options(load_only('lat', 'lon')))
found = set([stat.hashkey() for stat in stat_iter])
new_stat_values = []
for key in (wanted - found):
new_stat_values.append({
'lat': key.lat,
'lon': key.lon,
'time': today,
})
if new_stat_values:
# do a batch insert of new stats
stmt = MapStat.__table__.insert(
mysql_on_duplicate='id = id' # no-op
)
# but limit the batch depending on the model
ins_batch = MapStat._insert_batch
for i in range(0, len(new_stat_values), ins_batch):
batch_values = new_stat_values[i:i + ins_batch]
self.session.execute(stmt.values(batch_values))
if queue.size() >= batch:
self.task.apply_async(
kwargs={'batch': batch},
countdown=2,
expires=10)
return len(positions)
def test_fields(self):
today = util.utcnow().date()
session = self.session
session.add(MapStat(lat=12345, lon=-23456, time=today))
session.flush()
result = session.query(MapStat).first()
self.assertEqual(result.lat, 12345)
self.assertEqual(result.lon, -23456)
self.assertEqual(result.time, today)
def test_fields(self):
today = util.utcnow().date()
stat = MapStat(lat=12345, lon=-23456, time=today)
session = self.db_master_session
session.add(stat)
session.commit()
result = session.query(stat.__class__).first()
self.assertEqual(result.lat, 12345)
self.assertEqual(result.lon, -23456)
self.assertEqual(result.time, today)
def test_export_to_csv(self):
session = self.db_master_session
data = [
MapStat(lat=12345, lon=12345),
MapStat(lat=-10000, lon=-11000),
]
session.add_all(data)
session.flush()
fd, filename = mkstemp()
try:
result = export_to_csv(session, filename, multiplier=3)
self.assertEqual(result, 6)
written = os.read(fd, 10240)
lines = written.split()
self.assertEqual(len(lines), 6)
self.assertEqual(set([l[:6] for l in lines[:3]]), set(['12.345']))
self.assertEqual(set([l[:6] for l in lines[3:]]), set(['-9.999']))
finally:
os.remove(filename)
def test_export_to_csv(self):
session = self.session
data = [
MapStat(lat=12345, lon=12345),
MapStat(lat=0, lon=12345),
MapStat(lat=0, lon=0),
MapStat(lat=-10000, lon=-11000),
]
session.add_all(data)
session.flush()
fd, filename = mkstemp()
try:
result = export_to_csv(session, filename, multiplier=3)
self.assertEqual(result, 9)
written = os.read(fd, 10240)
lines = [line.split(',') for line in written.split()]
self.assertEqual(len(lines), 9)
self.assertEqual(set([round(float(l[0]), 2) for l in lines]),
set([-10.0, 0.0, 12.35]))
self.assertEqual(set([round(float(l[1]), 2) for l in lines]),
set([-11.0, 12.35]))
finally:
os.remove(filename)
def test_mapstat(self):
app = self.app
long_ago = date(2011, 10, 20)
today = util.utcnow().date()
session = self.db_master_session
stats = [
MapStat(lat=1000, lon=2000, time=long_ago),
MapStat(lat=2000, lon=3000, time=long_ago),
MapStat(lat=3000, lon=4000, time=long_ago),
]
session.add_all(stats)
session.flush()
app.post_json('/v1/submit', {
"items": [
{
"lat": 1.0,
"lon": 2.0,
"wifi": [{
"key": "aaaaaaaaaaaa"
}]
},
{
"lat": 2.0,
"lon": 3.0,
"wifi": [{
"key": "bbbbbbbbbbbb"
}]
},
{
"lat": 2.0,
"lon": 3.0,
"wifi": [{
"key": "cccccccccccc"
}]
},
{
"lat": -2.0,
"lon": 3.0,
"wifi": [{
"key": "cccccccccccc"
}]
},
{
"lat": 10.0,
"lon": 10.0,
"wifi": [{
"key": "invalid"
}]
},
]
},
status=204)
# check coarse grained stats
result = session.query(MapStat).all()
self.assertEqual(len(result), 4)
self.assertEqual(
sorted([(int(r.lat), int(r.lon), r.time, r.id) for r in result]), [
(-2000, 3000, today, stats[2].id + 1),
(1000, 2000, long_ago, stats[0].id),
(2000, 3000, long_ago, stats[1].id),
(3000, 4000, long_ago, stats[2].id),
])
