def test_leaders(self):
session = self.db_master_session
today = util.utcnow().date()
yesterday = today - timedelta(days=1)
for i in range(7, 1, -1):
user = User(nickname=unicode(i))
session.add(user)
session.flush()
score1 = Score(key=ScoreKey.location,
userid=user.id, time=today, value=i)
session.add(score1)
score2 = Score(key=ScoreKey.location,
userid=user.id, time=yesterday, value=i + 1)
session.add(score2)
session.commit()
request = DummyRequest()
request.db_slave_session = self.db_master_session
request.registry.redis_client = self.redis_client
inst = self._make_view(request)
result = inst.leaders_view()
self.assertEqual(
result['leaders1'],
[{'anchor': u'7', 'nickname': u'7', 'num': 15, 'pos': 1},
{'anchor': u'6', 'nickname': u'6', 'num': 13, 'pos': 2}])
self.assertEqual(
result['leaders2'],
[{'anchor': u'5', 'nickname': u'5', 'num': 11, 'pos': 3}])
# call the view again, without a working db session, so
# we can be sure to use the cached result
inst = self._make_view(request)
request.db_slave_session = None
second_result = inst.leaders_view()
self.assertEqual(second_result, result)
def test_leaders_weekly(self):
session = self.db_master_session
today = util.utcnow().date()
for i in range(3):
user = User(nickname=unicode(i))
session.add(user)
session.flush()
score1 = Score(key=ScoreKey.new_cell,
userid=user.id, time=today, value=i)
session.add(score1)
score2 = Score(key=ScoreKey.new_wifi,
userid=user.id, time=today, value=i)
session.add(score2)
session.commit()
request = DummyRequest()
request.db_slave_session = self.db_master_session
request.registry.redis_client = self.redis_client
inst = self._make_view(request)
result = inst.leaders_weekly_view()
for score_name in ('new_cell', 'new_wifi'):
self.assertEqual(
result['scores'][score_name]['leaders1'],
[{'nickname': u'2', 'num': 2, 'pos': 1},
{'nickname': u'1', 'num': 1, 'pos': 2}])
self.assertEqual(
result['scores'][score_name]['leaders2'],
[{'nickname': u'0', 'num': 0, 'pos': 3}])
# call the view again, without a working db session, so
# we can be sure to use the cached result
inst = self._make_view(request)
request.db_slave_session = None
second_result = inst.leaders_weekly_view()
self.assertEqual(second_result, result)
def test_leaders(self):
session = self.db_master_session
today = util.utcnow().date()
test_data = []
for i in range(20):
test_data.append((u'nick-%s' % i, 30))
highest = u'nick-high-too-long_'
highest += (128 - len(highest)) * u'x'
test_data.append((highest, 40))
lowest = u'nick-low'
test_data.append((lowest, 20))
for nick, value in test_data:
user = User(nickname=nick)
session.add(user)
session.flush()
score = Score(key=ScoreKey.location,
userid=user.id,
time=today,
value=value)
session.add(score)
session.commit()
# check the result
result = leaders(session)
self.assertEqual(len(result), 22)
self.assertEqual(result[0]['nickname'], highest[:24] + u'...')
self.assertEqual(result[0]['num'], 40)
self.assertTrue(lowest in [r['nickname'] for r in result])
def __call__(self, batch=1000):
score_values = defaultdict(int)
for score in self.queue.dequeue(batch=batch):
key = score["hashkey"]
if key.time is None:
key.time = self.today
score_values[key] += score["value"]
score_iter = Score.iterkeys(
self.session, list(score_values.keys()), extra=lambda query: query.options(load_only("value"))
)
scores = {}
for score in score_iter:
scores[score.hashkey()] = score
for key, value in score_values.items():
score = scores.get(key, None)
value = int(value)
if score is not None:
score.value += value
else:
stmt = Score.__table__.insert(mysql_on_duplicate="value = value + %s" % value).values(
value=value, **key.__dict__
)
self.session.execute(stmt)
if self.queue.size() >= batch:
self.task.apply_async(kwargs={"batch": batch}, countdown=2, expires=10)
return len(scores)
def _queue(self, triples):
scores = [{
'hashkey':
Score.to_hashkey(userid=userid, key=key, time=None),
'value':
value
} for userid, key, value in triples]
self.queue.enqueue(scores)
def test_enum(self):
session = self.session
session.add(Score(key=ScoreKey.location, value=13))
session.flush()
result = session.query(Score).first()
self.assertEqual(result.key, ScoreKey.location)
self.assertEqual(int(result.key), 0)
self.assertEqual(result.key.name, 'location')
def update(self, batch=1000):
scores = defaultdict(int)
for score in self.queue.dequeue(batch=batch):
key = score['hashkey']
if key.time is None:
key.time = self.today
scores[key] += score['value']
for key, value in scores.items():
Score.incr(self.session, key, value)
if self.queue.size() >= batch:
self.task.apply_async(
kwargs={'batch': batch},
countdown=1,
expires=57)
return len(scores)
def test_enum(self):
utcday = util.utcnow().date()
self.session.add(Score(
key=ScoreKey.location, userid=3, time=utcday, value=13))
self.session.flush()
result = self.session.query(Score).first()
self.assertEqual(result.key, ScoreKey.location)
self.assertEqual(int(result.key), 0)
self.assertEqual(result.key.name, 'location')
def test_enum(self):
score = Score(key=ScoreKey.location, value=13)
session = self.db_master_session
session.add(score)
session.commit()
result = session.query(score.__class__).first()
self.assertEqual(result.key, ScoreKey.location)
self.assertEqual(int(result.key), 0)
self.assertEqual(result.key.name, 'location')
def test_leaders_weekly(self):
session = self.db_master_session
today = util.utcnow().date()
test_data = []
for i in range(1, 11):
test_data.append((u'nick-%s' % i, i))
for nick, value in test_data:
user = User(nickname=nick)
session.add(user)
session.flush()
score = Score(key=ScoreKey.new_cell,
userid=user.id,
time=today,
value=value)
session.add(score)
score = Score(key=ScoreKey.new_wifi,
userid=user.id,
time=today,
value=21 - value)
session.add(score)
session.commit()
# check the result
result = leaders_weekly(session, batch=5)
self.assertEqual(len(result), 2)
self.assertEqual(set(result.keys()), set(['new_cell', 'new_wifi']))
# check the cell scores
scores = result['new_cell']
self.assertEqual(len(scores), 5)
self.assertEqual(scores[0]['nickname'], 'nick-10')
self.assertEqual(scores[0]['num'], 10)
self.assertEqual(scores[-1]['nickname'], 'nick-6')
self.assertEqual(scores[-1]['num'], 6)
# check the wifi scores
scores = result['new_wifi']
self.assertEqual(len(scores), 5)
self.assertEqual(scores[0]['nickname'], 'nick-1')
self.assertEqual(scores[0]['num'], 20)
self.assertEqual(scores[-1]['nickname'], 'nick-5')
self.assertEqual(scores[-1]['num'], 16)
def test_fields(self):
utcday = util.utcnow().date()
self.session.add(Score(
key=ScoreKey.location, userid=3, time=utcday, value=15))
self.session.flush()
result = self.session.query(Score).first()
self.assertEqual(result.key, ScoreKey.location)
self.assertEqual(result.userid, 3)
self.assertEqual(result.time, utcday)
self.assertEqual(result.value, 15)
def test_fields(self):
utcday = util.utcnow().date()
score = Score(key=ScoreKey.location, userid=3, time=utcday, value=15)
session = self.db_master_session
session.add(score)
session.commit()
result = session.query(score.__class__).first()
self.assertEqual(result.key, ScoreKey.location)
self.assertEqual(result.userid, 3)
self.assertEqual(result.time, utcday)
self.assertEqual(result.value, 15)
def test_nickname_header_update(self):
app = self.app
nickname = 'World Tr\xc3\xa4veler'
utcday = util.utcnow().date()
session = self.db_master_session
user = User(nickname=nickname.decode('utf-8'))
session.add(user)
session.flush()
session.add(
Score(key=ScoreKey.location, userid=user.id, time=utcday, value=7))
session.add(
Score(key=ScoreKey.new_wifi, userid=user.id, time=utcday, value=3))
session.commit()
app.post_json('/v1/submit', {
"items": [
{
"lat": 1.0,
"lon": 2.0,
"wifi": [{
"key": "00AAAAAAAAAA"
}]
},
]
},
headers={'X-Nickname': nickname},
status=204)
result = session.query(User).all()
self.assertEqual(len(result), 1)
self.assertEqual(result[0].nickname, nickname.decode('utf-8'))
result = session.query(Score).all()
self.assertEqual(len(result), 2)
self.assertEqual(set([r.key.name for r in result]),
set(['location', 'new_wifi']))
for r in result:
if r.key.name == 'location':
self.assertEqual(r.value, 8)
self.assertEqual(r.time, utcday)
elif r.key.name == 'new_wifi':
self.assertEqual(r.value, 4)
self.assertEqual(r.time, utcday)
def _add(self, quads):
for userid, key, time, value in quads:
self.session.add(
Score(userid=userid, key=key, time=time, value=value))
self.session.flush()
def test_insert_observations(self):
session = self.db_master_session
time = util.utcnow() - timedelta(days=1)
today = util.utcnow().date()
session.add(Wifi(key="ab1234567890", new_measures=0, total_measures=0))
session.add(Score(key=ScoreKey.new_wifi, userid=1, time=today,
value=7))
session.flush()
obs = dict(
created=time,
lat=1.0,
lon=2.0,
time=time,
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=-1,
heading=52.9,
speed=158.5,
)
entries = [
{
"key": "ab1234567890",
"channel": 11,
"signal": -80
},
{
"key": "ab1234567890",
"channel": 3,
"signal": -90
},
{
"key": "ab1234567890",
"channel": 3,
"signal": -80
},
{
"key": "cd3456789012",
"channel": 3,
"signal": -90
},
]
for e in entries:
e.update(obs)
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
observations = session.query(WifiObservation).all()
self.assertEqual(len(observations), 4)
self.assertEqual(set([o.key for o in observations]),
set(["ab1234567890", "cd3456789012"]))
self.assertEqual(set([o.channel for o in observations]), set([3, 11]))
self.assertEqual(set([o.signal for o in observations]), set([-80,
-90]))
self.assertEqual(set([o.heading or o in observations]), set([52.9]))
self.assertEqual(set([o.speed or o in observations]), set([158.5]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self.assertEqual(set([w.key for w in wifis]),
set(["ab1234567890", "cd3456789012"]))
self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, ScoreKey.new_wifi)
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
observations = session.query(WifiObservation).all()
self.assertEqual(len(observations), 8)
def test_insert_observations(self):
session = self.db_master_session
time = util.utcnow() - timedelta(days=1)
today = util.utcnow().date()
mcc = FRANCE_MCC
session.add(
Cell(radio=RADIO_TYPE['gsm'],
mcc=mcc,
mnc=2,
lac=3,
cid=4,
psc=5,
new_measures=2,
total_measures=5))
session.add(Score(key=ScoreKey.new_cell, userid=1, time=today,
value=7))
session.flush()
obs = dict(
created=time,
lat=PARIS_LAT,
lon=PARIS_LON,
time=time,
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE['gsm'],
)
entries = [
# Note that this first entry will be skipped as it does
# not include (lac, cid) or (psc)
{
"mcc": mcc,
"mnc": 2,
"signal": -100
},
{
"mcc": mcc,
"mnc": 2,
"lac": 3,
"cid": 4,
"psc": 5,
"asu": 8
},
{
"mcc": mcc,
"mnc": 2,
"lac": 3,
"cid": 4,
"psc": 5,
"asu": 8
},
{
"mcc": mcc,
"mnc": 2,
"lac": 3,
"cid": 4,
"psc": 5,
"asu": 15
},
{
"mcc": mcc,
"mnc": 2,
"lac": 3,
"cid": 7,
"psc": 5
},
]
for e in entries:
e.update(obs)
result = insert_measures_cell.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
observations = session.query(CellObservation).all()
self.assertEqual(len(observations), 4)
self.assertEqual(set([o.mcc for o in observations]), set([mcc]))
self.assertEqual(set([o.mnc for o in observations]), set([2]))
self.assertEqual(set([o.asu for o in observations]), set([-1, 8, 15]))
self.assertEqual(set([o.psc for o in observations]), set([5]))
self.assertEqual(set([o.signal for o in observations]), set([0]))
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.assertEqual(set([c.mcc for c in cells]), set([mcc]))
self.assertEqual(set([c.mnc for c in cells]), set([2]))
self.assertEqual(set([c.lac for c in cells]), set([3]))
self.assertEqual(set([c.cid for c in cells]), set([4, 7]))
self.assertEqual(set([c.psc for c in cells]), set([5]))
self.assertEqual(set([c.new_measures for c in cells]), set([1, 5]))
self.assertEqual(set([c.total_measures for c in cells]), set([1, 8]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, ScoreKey.new_cell)
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_measures_cell.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
observations = session.query(CellObservation).all()
self.assertEqual(len(observations), 8)
def test_insert_observations_invalid_lac(self):
session = self.db_master_session
schema = ValidCellKeySchema()
time = util.utcnow() - timedelta(days=1)
today = util.utcnow().date()
session.add(
Cell(radio=RADIO_TYPE['gsm'],
mcc=FRANCE_MCC,
mnc=2,
lac=3,
cid=4,
new_measures=2,
total_measures=5))
session.add(Score(key=ScoreKey.new_cell, userid=1, time=today,
value=7))
session.flush()
obs = dict(created=time,
lat=PARIS_LAT,
lon=PARIS_LON,
time=time,
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE['gsm'])
entries = [
{
"mcc": FRANCE_MCC,
"mnc": 2,
"lac": constants.MAX_LAC_ALL + 1,
"cid": constants.MAX_CID_ALL + 1,
"psc": 5,
"asu": 8
},
{
"mcc": FRANCE_MCC,
"mnc": 2,
"lac": schema.fields['lac'].missing,
"cid": schema.fields['cid'].missing,
"psc": 5,
"asu": 8
},
]
for e in entries:
e.update(obs)
result = insert_measures_cell.delay(entries, userid=1)
self.assertEqual(result.get(), 2)
observations = session.query(CellObservation).all()
self.assertEqual(len(observations), 2)
self.assertEqual(set([o.lac for o in observations]),
set([schema.fields['lac'].missing]))
self.assertEqual(set([o.cid for o in observations]),
set([schema.fields['cid'].missing]))
# Nothing should change in the initially created Cell record
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
self.assertEqual(set([c.new_measures for c in cells]), set([2]))
self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def _queue(self, triples):
scores = [{'hashkey': Score.to_hashkey(userid=userid,
key=key,
time=None),
'value': value} for userid, key, value in triples]
self.queue.enqueue(scores)
