def test_insert_measures_overflow(self):
session = self.db_master_session
key = "001234567890"
measures = [dict(id=0,
key=key,
lat=1.0 + i * 0.0000001,
lon=2.0 + i * 0.0000001) for i in range(3)]
result = insert_measures_wifi.delay(measures)
self.assertEqual(result.get(), 3)
result = insert_measures_wifi.delay(measures, max_measures_per_wifi=3)
self.assertEqual(result.get(), 0)
result = insert_measures_wifi.delay(measures, max_measures_per_wifi=10)
self.assertEqual(result.get(), 3)
result = insert_measures_wifi.delay(measures, max_measures_per_wifi=3)
self.assertEqual(result.get(), 0)
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 6)
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self.assertEqual(wifis[0].total_measures, 6)
def test_database_error(self):
session = self.session
stmt = text('drop table wifi;')
session.execute(stmt)
entries = [
{'lat': 1.0, 'lon': 2.0,
'key': 'ab:12:34:56:78:90', 'channel': 11},
{'lat': 1.0, 'lon': 2.0,
'key': 'ab:12:34:56:78:90', 'channel': 3},
{'lat': 1.0, 'lon': 2.0,
'key': 'ab:12:34:56:78:90', 'channel': 3},
{'lat': 1.0, 'lon': 2.0,
'key': 'cd:12:34:56:78:90', 'channel': 3},
]
try:
insert_measures_wifi.delay(entries)
except ProgrammingError:
pass
except Exception as exc:
self.fail('Unexpected exception caught: %s' % repr(exc))
self.check_raven([('ProgrammingError', 1)])
self.check_statcounter(StatKey.wifi, 0)
self.check_statcounter(StatKey.unique_wifi, 0)
def test_database_error(self):
session = self.db_master_session
stmt = text("drop table wifi;")
session.execute(stmt)
entries = [
{"lat": 1.0, "lon": 2.0,
"key": "ab:12:34:56:78:90", "channel": 11},
{"lat": 1.0, "lon": 2.0,
"key": "ab:12:34:56:78:90", "channel": 3},
{"lat": 1.0, "lon": 2.0,
"key": "ab:12:34:56:78:90", "channel": 3},
{"lat": 1.0, "lon": 2.0,
"key": "cd:12:34:56:78:90", "channel": 3},
]
try:
insert_measures_wifi.delay(entries)
except ProgrammingError:
pass
except Exception as exc:
self.fail("Unexpected exception caught: %s" % repr(exc))
find_msg = self.find_heka_messages
messages = find_msg('sentry', RAVEN_ERROR, field_name='msg')
self.assertEquals(len(messages), 1)
payload = messages[0].payload
# duplicate raven.base.RavenClient.decode
data = json.loads(zlib.decompress(base64.b64decode(payload)))
sentry_exc = data['sentry.interfaces.Exception']
self.assertEqual(sentry_exc['module'], ProgrammingError.__module__)
self.assertEqual(sentry_exc['type'], 'ProgrammingError')
def test_insert_measures_overflow(self):
session = self.db_master_session
key = "001234567890"
measures = [
dict(key=key, lat=1.0 + i * 0.0000001, lon=2.0 + i * 0.0000001)
for i in range(3)
]
result = insert_measures_wifi.delay(measures)
self.assertEqual(result.get(), 3)
result = insert_measures_wifi.delay(measures, max_measures_per_wifi=3)
self.assertEqual(result.get(), 0)
result = insert_measures_wifi.delay(measures, max_measures_per_wifi=10)
self.assertEqual(result.get(), 3)
result = insert_measures_wifi.delay(measures, max_measures_per_wifi=3)
self.assertEqual(result.get(), 0)
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 6)
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self.assertEqual(wifis[0].total_measures, 6)
def test_database_error(self):
session = self.session
stmt = text("drop table wifi;")
session.execute(stmt)
entries = [
{"lat": 1.0, "lon": 2.0,
"key": "ab:12:34:56:78:90", "channel": 11},
{"lat": 1.0, "lon": 2.0,
"key": "ab:12:34:56:78:90", "channel": 3},
{"lat": 1.0, "lon": 2.0,
"key": "ab:12:34:56:78:90", "channel": 3},
{"lat": 1.0, "lon": 2.0,
"key": "cd:12:34:56:78:90", "channel": 3},
]
try:
insert_measures_wifi.delay(entries)
except ProgrammingError:
pass
except Exception as exc:
self.fail("Unexpected exception caught: %s" % repr(exc))
self.check_raven([('ProgrammingError', 4)])
def test_insert_observations(self):
session = self.session
time = util.utcnow() - timedelta(days=1)
today = util.utcnow().date()
session.add(Wifi(key="ab1234567890",
new_measures=0, total_measures=0))
session.add(Score(key=ScoreKey.new_wifi,
userid=1, time=today, value=7))
session.flush()
obs = dict(
created=time, lat=1.0, lon=2.0,
time=time, accuracy=0, altitude=0,
altitude_accuracy=0, radio=-1,
heading=52.9,
speed=158.5,
)
entries = [
{"key": "ab1234567890", "channel": 11, "signal": -80},
{"key": "ab1234567890", "channel": 3, "signal": -90},
{"key": "ab1234567890", "channel": 3, "signal": -80},
{"key": "cd3456789012", "channel": 3, "signal": -90},
]
for e in entries:
e.update(obs)
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
observations = session.query(WifiObservation).all()
self.assertEqual(len(observations), 4)
self.assertEqual(set([o.key for o in observations]),
set(["ab1234567890", "cd3456789012"]))
self.assertEqual(set([o.channel for o in observations]), set([3, 11]))
self.assertEqual(set([o.signal for o in observations]),
set([-80, -90]))
self.assertEqual(set([o.heading or o in observations]), set([52.9]))
self.assertEqual(set([o.speed or o in observations]), set([158.5]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890",
"cd3456789012"]))
self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, ScoreKey.new_wifi)
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
observations = session.query(WifiObservation).all()
self.assertEqual(len(observations), 8)
def test_insert_measures(self):
session = self.db_master_session
time = util.utcnow() - timedelta(days=1)
session.add(Wifi(key="ab1234567890"))
session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7))
session.flush()
measure = dict(
id=0, created=encode_datetime(time), lat=1.0, lon=2.0,
time=encode_datetime(time), accuracy=0, altitude=0,
altitude_accuracy=0, radio=-1,
heading=52.9,
speed=158.5,
)
entries = [
{"key": "ab1234567890", "channel": 11, "signal": -80},
{"key": "ab1234567890", "channel": 3, "signal": -90},
{"key": "ab1234567890", "channel": 3, "signal": -80},
{"key": "cd3456789012", "channel": 3, "signal": -90},
]
for e in entries:
e.update(measure)
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 4)
self.assertEqual(set([m.key for m in measures]), set(["ab1234567890",
"cd3456789012"]))
self.assertEqual(set([m.channel for m in measures]), set([3, 11]))
self.assertEqual(set([m.signal for m in measures]), set([-80, -90]))
self.assertEqual(set([m.heading or m in measures]), set([52.9]))
self.assertEqual(set([m.speed or m in measures]), set([158.5]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890",
"cd3456789012"]))
self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi'])
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 8)
def test_database_error(self):
session = self.db_master_session
stmt = text("drop table wifi;")
session.execute(stmt)
entries = [
{
"lat": 1.0,
"lon": 2.0,
"key": "ab:12:34:56:78:90",
"channel": 11
},
{
"lat": 1.0,
"lon": 2.0,
"key": "ab:12:34:56:78:90",
"channel": 3
},
{
"lat": 1.0,
"lon": 2.0,
"key": "ab:12:34:56:78:90",
"channel": 3
},
{
"lat": 1.0,
"lon": 2.0,
"key": "cd:12:34:56:78:90",
"channel": 3
},
]
try:
insert_measures_wifi.delay(entries)
except ProgrammingError:
pass
except Exception as exc:
self.fail("Unexpected exception caught: %s" % repr(exc))
find_msg = self.find_heka_messages
messages = find_msg('sentry', RAVEN_ERROR, field_name='msg')
self.assertEquals(len(messages), 1)
payload = messages[0].payload
# duplicate raven.base.RavenClient.decode
data = json.loads(zlib.decompress(base64.b64decode(payload)))
sentry_exc = data['sentry.interfaces.Exception']
self.assertEqual(sentry_exc['module'], ProgrammingError.__module__)
self.assertEqual(sentry_exc['type'], 'ProgrammingError')
def test_blacklist(self):
utcnow = util.utcnow()
bad_wifi = WifiFactory.build()
good_wifi = WifiFactory.build()
black = WifiBlacklist(time=utcnow, count=1, key=bad_wifi.key)
self.session.add(black)
self.session.flush()
obs = dict(lat=good_wifi.lat, lon=good_wifi.lon)
entries = [
{'key': good_wifi.key},
{'key': good_wifi.key},
{'key': bad_wifi.key},
]
for entry in entries:
entry.update(obs)
result = insert_measures_wifi.delay(entries)
self.assertEqual(result.get(), 2)
self.assertEqual(self.data_queue.size(), 2)
update_wifi.delay().get()
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self._compare_sets([w.key for w in wifis], [good_wifi.key])
self.check_statcounter(StatKey.wifi, 2)
self.check_statcounter(StatKey.unique_wifi, 1)
def test_blacklist_time_used_as_creation_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1)
session = self.session
wifi_key = "ab1234567890"
session.add(WifiBlacklist(time=last_week, count=1, key=wifi_key))
session.flush()
# add a new entry for the previously blacklisted wifi
obs = dict(lat=PARIS_LAT, lon=PARIS_LON, key=wifi_key)
insert_measures_wifi.delay([obs]).get()
# the wifi was inserted again
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
# and the creation date was set to the date of the blacklist entry
self.assertEqual(wifis[0].created, last_week)
def test_blacklist_time_used_as_creation_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1)
session = self.db_master_session
wifi_key = "ab1234567890"
session.add(WifiBlacklist(time=last_week, count=1, key=wifi_key))
session.flush()
# add a new entry for the previously blacklisted wifi
obs = dict(lat=PARIS_LAT, lon=PARIS_LON, key=wifi_key)
insert_measures_wifi.delay([obs]).get()
# the wifi was inserted again
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
# and the creation date was set to the date of the blacklist entry
self.assertEqual(wifis[0].created, last_week)
def test_database_error(self):
session = self.session
stmt = text("drop table wifi;")
session.execute(stmt)
entries = [
{
"lat": 1.0,
"lon": 2.0,
"key": "ab:12:34:56:78:90",
"channel": 11
},
{
"lat": 1.0,
"lon": 2.0,
"key": "ab:12:34:56:78:90",
"channel": 3
},
{
"lat": 1.0,
"lon": 2.0,
"key": "ab:12:34:56:78:90",
"channel": 3
},
{
"lat": 1.0,
"lon": 2.0,
"key": "cd:12:34:56:78:90",
"channel": 3
},
]
try:
insert_measures_wifi.delay(entries)
except ProgrammingError:
pass
except Exception as exc:
self.fail("Unexpected exception caught: %s" % repr(exc))
self.check_raven([('ProgrammingError', 4)])
def test_blacklist_time_used_as_creation_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1)
wifi = WifiFactory.build()
self.session.add(WifiBlacklist(time=last_week, count=1, key=wifi.key))
self.session.flush()
# add a new entry for the previously blacklisted wifi
obs = dict(lat=wifi.lat, lon=wifi.lon, key=wifi.key)
insert_measures_wifi.delay([obs]).get()
self.assertEqual(self.data_queue.size(), 1)
update_wifi.delay().get()
# the wifi was inserted again
wifis = self.session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
# and the creation date was set to the date of the blacklist entry
self.assertEqual(wifis[0].created, last_week)
self.check_statcounter(StatKey.unique_wifi, 0)
def test_database_error(self):
self.session.execute(text('drop table wifi;'))
wifi = WifiFactory.build()
wifi2 = WifiFactory.build()
entries = [
{'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 7},
{'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 3},
{'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 3},
{'lat': wifi2.lat, 'lon': wifi2.lon, 'key': wifi2.key},
]
try:
insert_measures_wifi.delay(entries)
except ProgrammingError:
pass
except Exception as exc:
self.fail('Unexpected exception caught: %s' % repr(exc))
self.check_raven([('ProgrammingError', 1)])
self.check_statcounter(StatKey.wifi, 0)
self.check_statcounter(StatKey.unique_wifi, 0)
def test_insert_observations(self):
session = self.session
time = util.utcnow() - timedelta(days=1)
wifi = WifiFactory(total_measures=0)
wifi2 = WifiFactory.build(total_measures=0)
user = User(nickname=u'test')
session.add(user)
session.flush()
obs = dict(
created=time, time=time, lat=wifi.lat, lon=wifi.lon,
accuracy=None, altitude=None, altitude_accuracy=None,
heading=52.9, speed=158.5,
)
entries = [
{'key': wifi.key, 'channel': 11, 'signal': -80},
{'key': wifi.key, 'channel': 3, 'signal': -90},
{'key': wifi.key, 'channel': 3, 'signal': -80},
{'key': wifi2.key, 'channel': 3, 'signal': -90},
]
for entry in entries:
entry.update(obs)
result = insert_measures_wifi.delay(entries, userid=user.id)
self.assertEqual(result.get(), 4)
self.assertEqual(self.data_queue.size(), 4)
update_wifi.delay().get()
self.session.refresh(wifi)
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self._compare_sets([w.key for w in wifis], [wifi.key, wifi2.key])
self._compare_sets([w.total_measures for w in wifis], [1, 3])
score_queue = self.celery_app.data_queues['update_score']
scores = score_queue.dequeue()
self.assertEqual(len(scores), 1)
score = scores[0]
self.assertEqual(score['hashkey'].userid, user.id)
self.assertEqual(score['hashkey'].key, ScoreKey.new_wifi)
self.assertEqual(score['value'], 1)
self.check_statcounter(StatKey.wifi, 4)
self.check_statcounter(StatKey.unique_wifi, 1)
def test_blacklist(self):
session = self.db_master_session
bad_key = "ab1234567890"
good_key = "cd1234567890"
black = WifiBlacklist(key=bad_key)
session.add(black)
session.flush()
measure = dict(lat=1, lon=2)
entries = [{"key": good_key}, {"key": good_key}, {"key": bad_key}]
for e in entries:
e.update(measure)
result = insert_measures_wifi.delay(entries)
self.assertEqual(result.get(), 2)
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 2)
self.assertEqual(set([m.key for m in measures]), set([good_key]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self.assertEqual(set([w.key for w in wifis]), set([good_key]))
def test_blacklist(self):
utcnow = util.utcnow()
session = self.db_master_session
bad_key = "ab1234567890"
good_key = "cd1234567890"
black = WifiBlacklist(time=utcnow, count=1, key=bad_key)
session.add(black)
session.flush()
obs = dict(lat=1, lon=2)
entries = [{"key": good_key}, {"key": good_key}, {"key": bad_key}]
for e in entries:
e.update(obs)
result = insert_measures_wifi.delay(entries)
self.assertEqual(result.get(), 2)
observations = session.query(WifiObservation).all()
self.assertEqual(len(observations), 2)
self.assertEqual(set([o.key for o in observations]), set([good_key]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self.assertEqual(set([w.key for w in wifis]), set([good_key]))
def test_blacklist(self):
session = self.db_master_session
bad_key = "ab1234567890"
good_key = "cd1234567890"
black = WifiBlacklist(key=bad_key)
session.add(black)
session.flush()
measure = dict(id=0, lat=1, lon=2)
entries = [{"key": good_key}, {"key": good_key}, {"key": bad_key}]
for e in entries:
e.update(measure)
result = insert_measures_wifi.delay(entries)
self.assertEqual(result.get(), 2)
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 2)
self.assertEqual(
set([m.key for m in measures]), set([good_key]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self.assertEqual(set([w.key for w in wifis]), set([good_key]))
def test_blacklist(self):
utcnow = util.utcnow()
session = self.session
bad_key = "ab1234567890"
good_key = "cd1234567890"
black = WifiBlacklist(time=utcnow, count=1, key=bad_key)
session.add(black)
session.flush()
obs = dict(lat=1, lon=2)
entries = [{"key": good_key}, {"key": good_key}, {"key": bad_key}]
for e in entries:
e.update(obs)
result = insert_measures_wifi.delay(entries)
self.assertEqual(result.get(), 2)
observations = session.query(WifiObservation).all()
self.assertEqual(len(observations), 2)
self.assertEqual(
set([o.key for o in observations]), set([good_key]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 1)
self.assertEqual(set([w.key for w in wifis]), set([good_key]))
def test_blacklist_temporary_and_permanent(self):
session = self.db_master_session
# This test simulates a wifi that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
time_enc = encode_datetime(time)
measure = dict(id=month, key="ab1234567890",
time=time_enc,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-measures, override
# utcnow to set creation date
insert_result = insert_measures_wifi.delay(
[measure], utcnow=time_enc)
# update_result is (num-stations, num-moving-stations)
update_result = location_update_wifi.delay(min_new=1)
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = session.query(WifiBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# measure was supposedly inserted.
bl = bl[0]
bl.time = time
session.add(bl)
session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# measurement was admitted.
self.assertEqual(insert_result.get(), 1)
self.assertEqual(session.query(WifiMeasure).count(), month + 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# One wifi record should exist.
self.assertEqual(session.query(Wifi).count(), 1)
else:
# The station existed and was seen moving,
# thereby activating the blacklist.
self.assertEqual(update_result.get(), (1, 1))
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(session.query(WifiBlacklist).count(), 1)
self.assertEqual(session.query(Wifi).count(), 0)
# Try adding one more measurement 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = encode_datetime(time + timedelta(days=1))
measure['time'] = next_day
self.assertEqual(
0, insert_measures_wifi.delay([measure],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to "permanent" mode,
# so no measures accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), 0)
def test_blacklist_temporary_and_permanent(self):
session = self.db_master_session
# This test simulates a wifi that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
time_enc = encode_datetime(time)
measure = dict(key="ab1234567890",
time=time_enc,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-measures, override
# utcnow to set creation date
insert_result = insert_measures_wifi.delay([measure],
utcnow=time_enc)
# update_result is (num-stations, num-moving-stations)
update_result = location_update_wifi.delay(min_new=1)
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = session.query(WifiBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# measure was supposedly inserted.
bl = bl[0]
bl.time = time
session.add(bl)
session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# measurement was admitted.
self.assertEqual(insert_result.get(), 1)
self.assertEqual(session.query(WifiMeasure).count(), month + 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# One wifi record should exist.
self.assertEqual(session.query(Wifi).count(), 1)
else:
# The station existed and was seen moving,
# thereby activating the blacklist.
self.assertEqual(update_result.get(), (1, 1))
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(session.query(WifiBlacklist).count(), 1)
self.assertEqual(session.query(Wifi).count(), 0)
# Try adding one more measurement 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = encode_datetime(time + timedelta(days=1))
measure['time'] = next_day
self.assertEqual(
0,
insert_measures_wifi.delay([measure],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to "permanent" mode,
# so no measures accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), 0)
def test_insert_measures(self):
session = self.db_master_session
time = util.utcnow() - timedelta(days=1)
session.add(Wifi(key="ab1234567890"))
session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7))
session.flush()
measure = dict(
created=encode_datetime(time),
lat=1.0,
lon=2.0,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=-1,
heading=52.9,
speed=158.5,
)
entries = [
{
"key": "ab1234567890",
"channel": 11,
"signal": -80
},
{
"key": "ab1234567890",
"channel": 3,
"signal": -90
},
{
"key": "ab1234567890",
"channel": 3,
"signal": -80
},
{
"key": "cd3456789012",
"channel": 3,
"signal": -90
},
]
for e in entries:
e.update(measure)
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 4)
self.assertEqual(set([m.key for m in measures]),
set(["ab1234567890", "cd3456789012"]))
self.assertEqual(set([m.channel for m in measures]), set([3, 11]))
self.assertEqual(set([m.signal for m in measures]), set([-80, -90]))
self.assertEqual(set([m.heading or m in measures]), set([52.9]))
self.assertEqual(set([m.speed or m in measures]), set([158.5]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self.assertEqual(set([w.key for w in wifis]),
set(["ab1234567890", "cd3456789012"]))
self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi'])
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 8)
def test_insert_observations(self):
session = self.db_master_session
time = util.utcnow() - timedelta(days=1)
today = util.utcnow().date()
session.add(Wifi(key="ab1234567890", new_measures=0, total_measures=0))
session.add(Score(key=ScoreKey.new_wifi, userid=1, time=today,
value=7))
session.flush()
obs = dict(
created=time,
lat=1.0,
lon=2.0,
time=time,
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=-1,
heading=52.9,
speed=158.5,
)
entries = [
{
"key": "ab1234567890",
"channel": 11,
"signal": -80
},
{
"key": "ab1234567890",
"channel": 3,
"signal": -90
},
{
"key": "ab1234567890",
"channel": 3,
"signal": -80
},
{
"key": "cd3456789012",
"channel": 3,
"signal": -90
},
]
for e in entries:
e.update(obs)
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
observations = session.query(WifiObservation).all()
self.assertEqual(len(observations), 4)
self.assertEqual(set([o.key for o in observations]),
set(["ab1234567890", "cd3456789012"]))
self.assertEqual(set([o.channel for o in observations]), set([3, 11]))
self.assertEqual(set([o.signal for o in observations]), set([-80,
-90]))
self.assertEqual(set([o.heading or o in observations]), set([52.9]))
self.assertEqual(set([o.speed or o in observations]), set([158.5]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self.assertEqual(set([w.key for w in wifis]),
set(["ab1234567890", "cd3456789012"]))
self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, ScoreKey.new_wifi)
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_measures_wifi.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
observations = session.query(WifiObservation).all()
self.assertEqual(len(observations), 8)
