def add_measure(lat=0, lon=0, fail_counter=None, fails=10): try: if fail_counter: fail_counter[0] += 1 with add_measure.db_session() as session: measure = Measure(lat=lat, lon=lon) session.add(measure) if fail_counter: session.flush() measure2 = Measure(lat=0, lon=0) # provoke error via duplicate id measure2.id = measure.id if fail_counter[0] < fails: session.add(measure2) session.commit() except Exception as exc: raise add_measure.retry(exc=exc)
def process_measures(items, session, userid=None): utcnow = datetime.datetime.utcnow().replace(tzinfo=iso8601.UTC) utcmin = utcnow - datetime.timedelta(60) # get enough auto-increment ids assigned measures = [] for i in range(len(items)): measure = Measure() measures.append(measure) session.add(measure) # TODO switch unique measure id to a uuid, so we don't have to do # get these from a savepoint here session.flush() positions = [] cell_measures = [] wifi_measures = [] for i, item in enumerate(items): item = process_time(item, utcnow, utcmin) cell, wifi = process_measure(measures[i].id, item, session) cell_measures.extend(cell) wifi_measures.extend(wifi) positions.append({ 'lat': to_precise_int(item['lat']), 'lon': to_precise_int(item['lon']), }) heka_client = get_heka_client() if cell_measures: # group by and create task per cell key heka_client.incr("items.uploaded.cell_measures", len(cell_measures)) cells = defaultdict(list) for measure in cell_measures: cells[to_cellkey_psc(measure)].append(measure) for values in cells.values(): insert_cell_measures.delay(values, userid=userid) if wifi_measures: # group by and create task per wifi key heka_client.incr("items.uploaded.wifi_measures", len(wifi_measures)) wifis = defaultdict(list) for measure in wifi_measures: wifis[measure['key']].append(measure) for values in wifis.values(): insert_wifi_measures.delay(values, userid=userid) if userid is not None: process_score(userid, len(items), session) if positions: process_mapstat(positions, session, userid=userid)
def test_histogram(self): from ichnaea.content.tasks import histogram session = self.db_master_session today = datetime.utcnow().date() yesterday = (today - timedelta(1)) two_days = (today - timedelta(2)) long_ago = (today - timedelta(40)) wifi = '[{"key": "a"}]' measures = [ Measure(lat=10000000, lon=20000000, created=today, wifi=wifi), Measure(lat=10000000, lon=20000000, created=today, wifi=wifi), Measure(lat=10000000, lon=20000000, created=yesterday, wifi=wifi), Measure(lat=10000000, lon=20000000, created=two_days, wifi=wifi), Measure(lat=10000000, lon=20000000, created=two_days, wifi=wifi), Measure(lat=10000000, lon=20000000, created=two_days, wifi=wifi), Measure(lat=10000000, lon=20000000, created=long_ago, wifi=wifi), ] session.add_all(measures) session.commit() histogram.delay(ago=40).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 1) self.assertEqual(stats[0].key, STAT_TYPE['location']) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) # fill up newer dates histogram.delay(ago=2).get() histogram.delay(ago=1).get() histogram.delay(ago=0).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 4) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) self.assertEqual(stats[1].time, two_days) self.assertEqual(stats[1].value, 4) self.assertEqual(stats[2].time, yesterday) self.assertEqual(stats[2].value, 5) self.assertEqual(stats[3].time, today) self.assertEqual(stats[3].value, 7) # test duplicate execution result = histogram.delay(ago=1) self.assertEqual(result.get(), 0)
def process_measure(data, utcnow, session, userid=None): measure = Measure() measure.created = utcnow measure.time = data['time'] measure.lat = to_precise_int(data['lat']) measure.lon = to_precise_int(data['lon']) measure.accuracy = data['accuracy'] measure.altitude = data['altitude'] measure.altitude_accuracy = data['altitude_accuracy'] measure.radio = RADIO_TYPE.get(data['radio'], -1) # get measure.id set session.add(measure) session.flush() measure_data = dict( id=measure.id, created=encode_datetime(measure.created), lat=measure.lat, lon=measure.lon, time=encode_datetime(measure.time), accuracy=measure.accuracy, altitude=measure.altitude, altitude_accuracy=measure.altitude_accuracy, radio=measure.radio, ) if data.get('cell'): insert_cell_measure.delay(measure_data, data['cell'], userid=userid) measure.cell = dumps(data['cell']) if data.get('wifi'): # filter out old-style sha1 hashes too_long_keys = False for w in data['wifi']: w['key'] = key = normalize_wifi_key(w['key']) if len(key) > 12: too_long_keys = True break if not too_long_keys: insert_wifi_measure.delay( measure_data, data['wifi'], userid=userid) measure.wifi = dumps(data['wifi']) return measure
def _make_one(self, **kw): from ichnaea.models import Measure return Measure(**kw)
def process_measure(data, utcnow, session): session_objects = [] measure = Measure() measure.created = utcnow measure.time = data['time'] measure.lat = to_precise_int(data['lat']) measure.lon = to_precise_int(data['lon']) measure.accuracy = data['accuracy'] measure.altitude = data['altitude'] measure.altitude_accuracy = data['altitude_accuracy'] measure.radio = RADIO_TYPE.get(data['radio'], -1) # get measure.id set session.add(measure) session.flush() if data.get('cell'): cells, cell_data = process_cell(data['cell'], measure) measure.cell = dumps(cell_data) session_objects.extend(cells) if data.get('wifi'): # filter out old-style sha1 hashes too_long_keys = False for w in data['wifi']: w['key'] = key = normalize_wifi_key(w['key']) if len(key) > 12: too_long_keys = True break if not too_long_keys: process_wifi(data['wifi'], measure) measure.wifi = dumps(data['wifi']) return (measure, session_objects)