def add_measure(lat=0, lon=0, fail_counter=None, fails=10):
try:
if fail_counter:
fail_counter[0] += 1
with add_measure.db_session() as session:
measure = Measure(lat=lat, lon=lon)
session.add(measure)
if fail_counter:
session.flush()
measure2 = Measure(lat=0, lon=0)
# provoke error via duplicate id
measure2.id = measure.id
if fail_counter[0] < fails:
session.add(measure2)
session.commit()
except Exception as exc:
raise add_measure.retry(exc=exc)
def add_measure(lat=0, lon=0, fail_counter=None, fails=10):
try:
if fail_counter:
fail_counter[0] += 1
with add_measure.db_session() as session:
measure = Measure(lat=lat, lon=lon)
session.add(measure)
if fail_counter:
session.flush()
measure2 = Measure(lat=0, lon=0)
# provoke error via duplicate id
measure2.id = measure.id
if fail_counter[0] < fails:
session.add(measure2)
session.commit()
except Exception as exc:
raise add_measure.retry(exc=exc)
def process_measures(items, session, userid=None):
utcnow = datetime.datetime.utcnow().replace(tzinfo=iso8601.UTC)
utcmin = utcnow - datetime.timedelta(60)
# get enough auto-increment ids assigned
measures = []
for i in range(len(items)):
measure = Measure()
measures.append(measure)
session.add(measure)
# TODO switch unique measure id to a uuid, so we don't have to do
# get these from a savepoint here
session.flush()
positions = []
cell_measures = []
wifi_measures = []
for i, item in enumerate(items):
item = process_time(item, utcnow, utcmin)
cell, wifi = process_measure(measures[i].id, item, session)
cell_measures.extend(cell)
wifi_measures.extend(wifi)
positions.append({
'lat': to_precise_int(item['lat']),
'lon': to_precise_int(item['lon']),
})
heka_client = get_heka_client()
if cell_measures:
# group by and create task per cell key
heka_client.incr("items.uploaded.cell_measures",
len(cell_measures))
cells = defaultdict(list)
for measure in cell_measures:
cells[to_cellkey_psc(measure)].append(measure)
for values in cells.values():
insert_cell_measures.delay(values, userid=userid)
if wifi_measures:
# group by and create task per wifi key
heka_client.incr("items.uploaded.wifi_measures",
len(wifi_measures))
wifis = defaultdict(list)
for measure in wifi_measures:
wifis[measure['key']].append(measure)
for values in wifis.values():
insert_wifi_measures.delay(values, userid=userid)
if userid is not None:
process_score(userid, len(items), session)
if positions:
process_mapstat(positions, session, userid=userid)
def test_histogram(self):
from ichnaea.content.tasks import histogram
session = self.db_master_session
today = datetime.utcnow().date()
yesterday = (today - timedelta(1))
two_days = (today - timedelta(2))
long_ago = (today - timedelta(40))
wifi = '[{"key": "a"}]'
measures = [
Measure(lat=10000000, lon=20000000, created=today, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=today, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=yesterday, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=two_days, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=two_days, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=two_days, wifi=wifi),
Measure(lat=10000000, lon=20000000, created=long_ago, wifi=wifi),
]
session.add_all(measures)
session.commit()
histogram.delay(ago=40).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 1)
self.assertEqual(stats[0].key, STAT_TYPE['location'])
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
# fill up newer dates
histogram.delay(ago=2).get()
histogram.delay(ago=1).get()
histogram.delay(ago=0).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 4)
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
self.assertEqual(stats[1].time, two_days)
self.assertEqual(stats[1].value, 4)
self.assertEqual(stats[2].time, yesterday)
self.assertEqual(stats[2].value, 5)
self.assertEqual(stats[3].time, today)
self.assertEqual(stats[3].value, 7)
# test duplicate execution
result = histogram.delay(ago=1)
self.assertEqual(result.get(), 0)
def process_measure(data, utcnow, session, userid=None):
measure = Measure()
measure.created = utcnow
measure.time = data['time']
measure.lat = to_precise_int(data['lat'])
measure.lon = to_precise_int(data['lon'])
measure.accuracy = data['accuracy']
measure.altitude = data['altitude']
measure.altitude_accuracy = data['altitude_accuracy']
measure.radio = RADIO_TYPE.get(data['radio'], -1)
# get measure.id set
session.add(measure)
session.flush()
measure_data = dict(
id=measure.id, created=encode_datetime(measure.created),
lat=measure.lat, lon=measure.lon, time=encode_datetime(measure.time),
accuracy=measure.accuracy, altitude=measure.altitude,
altitude_accuracy=measure.altitude_accuracy,
radio=measure.radio,
)
if data.get('cell'):
insert_cell_measure.delay(measure_data, data['cell'], userid=userid)
measure.cell = dumps(data['cell'])
if data.get('wifi'):
# filter out old-style sha1 hashes
too_long_keys = False
for w in data['wifi']:
w['key'] = key = normalize_wifi_key(w['key'])
if len(key) > 12:
too_long_keys = True
break
if not too_long_keys:
insert_wifi_measure.delay(
measure_data, data['wifi'], userid=userid)
measure.wifi = dumps(data['wifi'])
return measure
def _make_one(self, **kw):
from ichnaea.models import Measure
return Measure(**kw)
def process_measure(data, utcnow, session):
session_objects = []
measure = Measure()
measure.created = utcnow
measure.time = data['time']
measure.lat = to_precise_int(data['lat'])
measure.lon = to_precise_int(data['lon'])
measure.accuracy = data['accuracy']
measure.altitude = data['altitude']
measure.altitude_accuracy = data['altitude_accuracy']
measure.radio = RADIO_TYPE.get(data['radio'], -1)
# get measure.id set
session.add(measure)
session.flush()
if data.get('cell'):
cells, cell_data = process_cell(data['cell'], measure)
measure.cell = dumps(cell_data)
session_objects.extend(cells)
if data.get('wifi'):
# filter out old-style sha1 hashes
too_long_keys = False
for w in data['wifi']:
w['key'] = key = normalize_wifi_key(w['key'])
if len(key) > 12:
too_long_keys = True
break
if not too_long_keys:
process_wifi(data['wifi'], measure)
measure.wifi = dumps(data['wifi'])
return (measure, session_objects)
