def add_measure(lat=0, lon=0, fail_counter=None, fails=10):
try:
if fail_counter:
fail_counter[0] += 1
with add_measure.db_session() as session:
measure = Measure(lat=lat, lon=lon)
session.add(measure)
if fail_counter:
session.flush()
measure2 = Measure(lat=0, lon=0)
# provoke error via duplicate id
measure2.id = measure.id
if fail_counter[0] < fails:
session.add(measure2)
session.commit()
except Exception as exc:
raise add_measure.retry(exc=exc)
def process_measures(items, session, userid=None):
utcnow = datetime.datetime.utcnow().replace(tzinfo=iso8601.UTC)
utcmin = utcnow - datetime.timedelta(60)
# get enough auto-increment ids assigned
measures = []
for i in range(len(items)):
measure = Measure()
measures.append(measure)
session.add(measure)
# TODO switch unique measure id to a uuid, so we don't have to do
# get these from a savepoint here
session.flush()
positions = []
cell_measures = []
wifi_measures = []
for i, item in enumerate(items):
item = process_time(item, utcnow, utcmin)
cell, wifi = process_measure(measures[i].id, item, session)
cell_measures.extend(cell)
wifi_measures.extend(wifi)
positions.append({
'lat': to_precise_int(item['lat']),
'lon': to_precise_int(item['lon']),
})
heka_client = get_heka_client()
if cell_measures:
# group by and create task per cell key
heka_client.incr("items.uploaded.cell_measures",
len(cell_measures))
cells = defaultdict(list)
for measure in cell_measures:
cells[to_cellkey_psc(measure)].append(measure)
for values in cells.values():
insert_cell_measures.delay(values, userid=userid)
if wifi_measures:
# group by and create task per wifi key
heka_client.incr("items.uploaded.wifi_measures",
len(wifi_measures))
wifis = defaultdict(list)
for measure in wifi_measures:
wifis[measure['key']].append(measure)
for values in wifis.values():
insert_wifi_measures.delay(values, userid=userid)
if userid is not None:
process_score(userid, len(items), session)
if positions:
process_mapstat(positions, session, userid=userid)
def test_histogram(self):
from ichnaea.content.tasks import histogram
session = self.db_master_session
today = datetime.utcnow().date()
yesterday = (today - timedelta(1))
two_days = (today - timedelta(2))
long_ago = (today - timedelta(40))
wifi = '[{"key": "a"}]'
measures = [
Measure(lat=10000000, lon=20000000, created=today, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=today, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=yesterday, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=two_days, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=two_days, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=two_days, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=long_ago, wifi=wifi),
]
session.add_all(measures)
session.commit()
histogram.delay(ago=40).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 1)
self.assertEqual(stats[0].key, STAT_TYPE['location'])
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
# fill up newer dates
histogram.delay(ago=2).get()
histogram.delay(ago=1).get()
histogram.delay(ago=0).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 4)
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
self.assertEqual(stats[1].time, two_days)
self.assertEqual(stats[1].value, 4)
self.assertEqual(stats[2].time, yesterday)
self.assertEqual(stats[2].value, 5)
self.assertEqual(stats[3].time, today)
self.assertEqual(stats[3].value, 7)
# test duplicate execution
result = histogram.delay(ago=1)
self.assertEqual(result.get(), 0)
def _make_one(self, **kw):
from ichnaea.models import Measure
return Measure(**kw)
