def test_leaders_weekly(self): session = self.db_master_session for i in range(3): user = User(nickname=unicode(i)) session.add(user) session.flush() score1 = Score(userid=user.id, value=i) score1.name = 'new_cell' session.add(score1) score2 = Score(userid=user.id, value=i) score2.name = 'new_wifi' session.add(score2) session.commit() request = DummyRequest() request.db_slave_session = self.db_master_session inst = self._make_view(request) result = inst.leaders_weekly_view() for score_name in ('new_cell', 'new_wifi'): self.assertEqual(result['scores'][score_name]['leaders1'], [{ 'nickname': u'2', 'num': 2, 'pos': 1 }, { 'nickname': u'1', 'num': 1, 'pos': 2 }]) self.assertEqual(result['scores'][score_name]['leaders2'], [{ 'nickname': u'0', 'num': 0, 'pos': 3 }])
def test_nickname_header_update(self): app = self.app nickname = 'World Tr\xc3\xa4veler' utcday = util.utcnow().date() session = self.db_master_session user = User(nickname=nickname.decode('utf-8')) session.add(user) session.flush() session.add(Score(userid=user.id, key=SCORE_TYPE['location'], value=7)) session.add(Score(userid=user.id, key=SCORE_TYPE['new_wifi'], value=3)) session.commit() app.post_json( '/v1/submit', {"items": [ {"lat": 1.0, "lon": 2.0, "wifi": [{"key": "00AAAAAAAAAA"}]}, ]}, headers={'X-Nickname': nickname}, status=204) result = session.query(User).all() self.assertEqual(len(result), 1) self.assertEqual(result[0].nickname, nickname.decode('utf-8')) result = session.query(Score).all() self.assertEqual(len(result), 2) self.assertEqual(set([r.name for r in result]), set(['location', 'new_wifi'])) for r in result: if r.name == 'location': self.assertEqual(r.value, 8) self.assertEqual(r.time, utcday) elif r.name == 'new_wifi': self.assertEqual(r.value, 4) self.assertEqual(r.time, utcday)
def test_leaders(self): session = self.db_master_session today = datetime.utcnow().date() yesterday = today - timedelta(days=1) for i in range(3): user = User(nickname=unicode(i)) session.add(user) session.flush() score1 = Score(userid=user.id, time=today, value=i) score1.name = 'location' session.add(score1) score2 = Score(userid=user.id, time=yesterday, value=i + 1) score2.name = 'location' session.add(score2) session.commit() request = DummyRequest() request.db_slave_session = self.db_master_session inst = self._make_view(request) result = inst.leaders_view() self.assertEqual( result['leaders1'], [{'nickname': u'2', 'num': 5}, {'nickname': u'1', 'num': 3}]) self.assertEqual( result['leaders2'], [{'nickname': u'0', 'num': 1}])
def test_leaders(self): from ichnaea.content.stats import leaders session = self.db_master_session test_data = [] for i in range(20): test_data.append((u'nick-%s' % i, 7)) highest = u'nick-high-too-long_' highest += (128 - len(highest)) * u'x' test_data.append((highest, 10)) lowest = u'nick-low' test_data.append((lowest, 5)) for nick, value in test_data: user = User(nickname=nick) session.add(user) session.flush() score = Score(userid=user.id, value=value) score.name = 'location' session.add(score) session.commit() # check the result result = leaders(session) self.assertEqual(len(result), 22) self.assertEqual(result[0]['nickname'], highest[:24] + u'...') self.assertEqual(result[0]['num'], 10) self.assertTrue(lowest in [r['nickname'] for r in result])
def test_insert_measures_invalid_lac(self): session = self.db_master_session schema = ValidCellBaseSchema() time = util.utcnow() - timedelta(days=1) session.add( Cell(radio=RADIO_TYPE['gsm'], mcc=FRANCE_MCC, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict(created=encode_datetime(time), lat=PARIS_LAT, lon=PARIS_LON, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm']) entries = [ { "mcc": FRANCE_MCC, "mnc": 2, "lac": constants.MAX_LAC_ALL + 1, "cid": constants.MAX_CID_ALL + 1, "psc": 5, "asu": 8 }, { "mcc": FRANCE_MCC, "mnc": 2, "lac": schema.fields['lac'].missing, "cid": schema.fields['cid'].missing, "psc": 5, "asu": 8 }, ] for e in entries: e.update(measure) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual(set([m.lac for m in measures]), set([schema.fields['lac'].missing])) self.assertEqual(set([m.cid for m in measures]), set([schema.fields['cid'].missing])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def test_insert_invalid_lac(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add( Cell(radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict(id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm']) entries = [ { "mcc": 1, "mnc": 2, "lac": 3147483647, "cid": 2147483647, "psc": 5, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": -1, "cid": -1, "psc": 5, "asu": 8 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual(set([m.lac for m in measures]), set([-1])) self.assertEqual(set([m.cid for m in measures]), set([-1])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def test_leaders(self): session = self.db_master_session today = util.utcnow().date() yesterday = today - timedelta(days=1) for i in range(7, 1, -1): user = User(nickname=unicode(i)) session.add(user) session.flush() score1 = Score(userid=user.id, time=today, value=i) score1.name = 'location' session.add(score1) score2 = Score(userid=user.id, time=yesterday, value=i + 1) score2.name = 'location' session.add(score2) session.commit() request = DummyRequest() request.db_slave_session = self.db_master_session request.registry.redis_client = self.redis_client inst = self._make_view(request) result = inst.leaders_view() self.assertEqual(result['leaders1'], [{ 'anchor': u'7', 'nickname': u'7', 'num': 15, 'pos': 1 }, { 'anchor': u'6', 'nickname': u'6', 'num': 13, 'pos': 2 }]) self.assertEqual(result['leaders2'], [{ 'anchor': u'5', 'nickname': u'5', 'num': 11, 'pos': 3 }]) # call the view again, without a working db session, so # we can be sure to use the cached result inst = self._make_view(request) request.db_slave_session = None second_result = inst.leaders_view() self.assertEqual(second_result, result)
def test_leaders_weekly(self): session = self.db_master_session for i in range(3): user = User(nickname=unicode(i)) session.add(user) session.flush() score1 = Score(userid=user.id, value=i) score1.name = 'new_cell' session.add(score1) score2 = Score(userid=user.id, value=i) score2.name = 'new_wifi' session.add(score2) session.commit() request = DummyRequest() request.db_slave_session = self.db_master_session request.registry.redis_client = self.redis_client inst = self._make_view(request) result = inst.leaders_weekly_view() for score_name in ('new_cell', 'new_wifi'): self.assertEqual(result['scores'][score_name]['leaders1'], [{ 'nickname': u'2', 'num': 2, 'pos': 1 }, { 'nickname': u'1', 'num': 1, 'pos': 2 }]) self.assertEqual(result['scores'][score_name]['leaders2'], [{ 'nickname': u'0', 'num': 0, 'pos': 3 }]) # call the view again, without a working db session, so # we can be sure to use the cached result inst = self._make_view(request) request.db_slave_session = None second_result = inst.leaders_weekly_view() self.assertEqual(second_result, result)
def test_leaders_weekly(self): from ichnaea.content.stats import leaders_weekly session = self.db_master_session test_data = [] for i in range(1, 11): test_data.append((u'nick-%s' % i, i)) for nick, value in test_data: user = User(nickname=nick) session.add(user) session.flush() score = Score(userid=user.id, value=value) score.name = 'new_cell' session.add(score) score = Score(userid=user.id, value=21 - value) score.name = 'new_wifi' session.add(score) session.commit() # check the result result = leaders_weekly(session, batch=5) self.assertEqual(len(result), 2) self.assertEqual(set(result.keys()), set(['new_cell', 'new_wifi'])) # check the cell scores scores = result['new_cell'] self.assertEqual(len(scores), 5) self.assertEqual(scores[0]['nickname'], 'nick-10') self.assertEqual(scores[0]['num'], 10) self.assertEqual(scores[-1]['nickname'], 'nick-6') self.assertEqual(scores[-1]['num'], 6) # check the wifi scores scores = result['new_wifi'] self.assertEqual(len(scores), 5) self.assertEqual(scores[0]['nickname'], 'nick-1') self.assertEqual(scores[0]['num'], 20) self.assertEqual(scores[-1]['nickname'], 'nick-5') self.assertEqual(scores[-1]['num'], 16)
def process_score(userid, points, session, key='location'): utcday = datetime.datetime.utcnow().date() key = SCORE_TYPE[key] rows = session.query(Score).filter(Score.userid == userid).filter( Score.key == key).filter( Score.time == utcday).limit(1).with_lockmode('update') old = rows.first() if old: # update score if isinstance(old.value, (int, long)): old.value = Score.value + points else: # already a sql expression old.value += points else: score = Score(userid=userid, key=key, time=utcday, value=points) session.add(score) return points
def test_insert_measures(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) session.add(Wifi(key="ab1234567890")) session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7)) session.flush() measure = dict( created=encode_datetime(time), lat=1.0, lon=2.0, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=-1, heading=52.9, speed=158.5, ) entries = [ { "key": "ab1234567890", "channel": 11, "signal": -80 }, { "key": "ab1234567890", "channel": 3, "signal": -90 }, { "key": "ab1234567890", "channel": 3, "signal": -80 }, { "key": "cd3456789012", "channel": 3, "signal": -90 }, ] for e in entries: e.update(measure) result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.key for m in measures]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([m.channel for m in measures]), set([3, 11])) self.assertEqual(set([m.signal for m in measures]), set([-80, -90])) self.assertEqual(set([m.heading or m in measures]), set([52.9])) self.assertEqual(set([m.speed or m in measures]), set([158.5])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3])) self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 8)
def test_insert_measures(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) mcc = FRANCE_MCC session.add( Cell(radio=RADIO_TYPE['gsm'], mcc=mcc, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( created=encode_datetime(time), lat=PARIS_LAT, lon=PARIS_LON, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) { "mcc": mcc, "mnc": 2, "signal": -100 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 7, "psc": 5 }, ] for e in entries: e.update(measure) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.mcc for m in measures]), set([mcc])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([5])) self.assertEqual(set([m.signal for m in measures]), set([0])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([mcc])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.psc for c in cells]), set([5])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_cell']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 8)
def test_wifi(self): from ichnaea.service.submit.tasks import insert_wifi_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add(Wifi(key="ab12")) session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=-1, ) entries = [ { "key": "ab12", "channel": 11, "signal": -80 }, { "key": "ab12", "channel": 3, "signal": -90 }, { "key": "ab12", "channel": 3, "signal": -80 }, { "key": "cd34", "channel": 3, "signal": -90 }, ] for e in entries: e.update(measure) result = insert_wifi_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.key for m in measures]), set(["ab12", "cd34"])) self.assertEqual(set([m.channel for m in measures]), set([3, 11])) self.assertEqual(set([m.signal for m in measures]), set([-80, -90])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self.assertEqual(set([w.key for w in wifis]), set(["ab12", "cd34"])) self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3])) self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_wifi_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 8)
def _make_one(self, **kw): from ichnaea.content.models import Score return Score(**kw)
def test_cell(self): from ichnaea.service.submit.tasks import insert_cell_measure session = self.db_master_session utcnow = datetime.utcnow() session.add( Cell(radio=0, mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(utcnow), lat=10000000, lon=20000000, time=encode_datetime(utcnow), accuracy=0, altitude=0, altitude_accuracy=0, radio=0, ) entries = [ { "mcc": 1, "mnc": 2, "signal": -100 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 15 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 7 }, ] result = insert_cell_measure.delay(measure, entries, userid=1) self.assertEqual(result.get(), 5) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 5) self.assertEqual(set([m.mcc for m in measures]), set([1])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([0, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([0, 5])) self.assertEqual(set([m.signal for m in measures]), set([0, -100])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([1])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_cell']) self.assertEqual(scores[0].value, 10) # test duplicate execution result = insert_cell_measure.delay(measure, entries, userid=1) self.assertEqual(result.get(), 5) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 10)