def trim_and_check(keep):
from ichnaea.content.tasks import get_curr_stat
# trim model to 'keep' measures per key
result = trim_func.delay(keep)
result.get()
# check that exactly the rows expected were kept
measures = session.query(measure_model).all()
self.assertEqual(len(measures), len(keys) * keep)
for k in keys:
for i in range(measures_per_key - keep / 2, measures_per_key):
self.assertTrue(
any(m.lat == m1 + i and m.lat == m1 + i and kcmp(m, k)
for m in measures))
self.assertTrue(
any(m.lat == m2 + i and m.lat == m2 + i and kcmp(m, k)
for m in measures))
stations = session.query(unique_model).all()
for s in stations:
self.assertTrue(s.new_measures <= s.total_measures)
# check that the deletion stat was updated
self.assertEqual(get_curr_stat(session, delstat),
len(keys) * (2 * measures_per_key - keep))
def trim_and_check(keep):
from ichnaea.content.tasks import get_curr_stat
# trim model to 'keep' measures per key
result = trim_func.delay(keep)
result.get()
# check that exactly the rows expected were kept
measures = session.query(measure_model).all()
self.assertEqual(len(measures), len(keys) * keep)
for k in keys:
for i in range(measures_per_key - keep / 2, measures_per_key):
self.assertTrue(any(m.lat == m1 + i and
m.lat == m1 + i and
kcmp(m, k)
for m in measures))
self.assertTrue(any(m.lat == m2 + i and
m.lat == m2 + i and
kcmp(m, k)
for m in measures))
stations = session.query(unique_model).all()
for s in stations:
self.assertTrue(s.new_measures <= s.total_measures)
# check that the deletion stat was updated
self.assertEqual(get_curr_stat(session, delstat),
len(keys) * (2 * measures_per_key - keep))
def check_no_trim_young_data(self, unique_model, measure_model, trim_func,
kinit, delstat):
"""
Check that a trim function run against young data leaves it alone.
"""
from ichnaea.content.tasks import get_curr_stat
session = self.db_master_session
measures = []
keys = range(3)
measures_per_key = 4
m1 = 10000000
m2 = 20000000
session.query(unique_model).delete()
session.query(measure_model).delete()
session.flush()
for k in keys:
kargs = kinit(k)
measures.append(
unique_model(lat=(m1 + m2) / 2,
lon=(m1 + m2) / 2,
total_measures=measures_per_key * 2,
**kargs))
for i in range(measures_per_key):
measures.append(measure_model(lat=m1 + i, lon=m1 + i, **kargs))
measures.append(measure_model(lat=m2 + i, lon=m2 + i, **kargs))
session.add_all(measures)
session.flush()
measures = session.query(measure_model).count()
self.assertEqual(measures, measures_per_key * 2 * len(keys))
dels = get_curr_stat(session, delstat)
result = trim_func.delay(2)
result.get()
# check that all data was preserved
measures = session.query(measure_model).count()
self.assertEqual(measures, measures_per_key * 2 * len(keys))
# check that the deletion stat is unchanged
self.assertEqual(get_curr_stat(session, delstat), dels)
def check_no_trim_young_data(self, unique_model, measure_model,
trim_func, kinit, delstat):
"""
Check that a trim function run against young data leaves it alone.
"""
from ichnaea.content.tasks import get_curr_stat
session = self.db_master_session
measures = []
keys = range(3)
measures_per_key = 4
m1 = 10000000
m2 = 20000000
session.query(unique_model).delete()
session.query(measure_model).delete()
session.flush()
for k in keys:
kargs = kinit(k)
measures.append(unique_model(lat=(m1 + m2) / 2, lon=(m1 + m2) / 2,
total_measures=measures_per_key * 2,
**kargs))
for i in range(measures_per_key):
measures.append(measure_model(lat=m1 + i, lon=m1 + i, **kargs))
measures.append(measure_model(lat=m2 + i, lon=m2 + i, **kargs))
session.add_all(measures)
session.flush()
measures = session.query(measure_model).count()
self.assertEqual(measures, measures_per_key * 2 * len(keys))
dels = get_curr_stat(session, delstat)
result = trim_func.delay(2)
result.get()
# check that all data was preserved
measures = session.query(measure_model).count()
self.assertEqual(measures, measures_per_key * 2 * len(keys))
# check that the deletion stat is unchanged
self.assertEqual(get_curr_stat(session, delstat), dels)
def test_incr_and_curr_stats(self):
"""
Check that ichnaea.content.tasks.{incr_stat,get_curr_stat}
work on the most-recent-stat associated with a given day,
and copy forward each day's stat to the next day, as they go.
"""
from ichnaea.content.tasks import incr_stat, get_curr_stat
session = self.db_master_session
today = datetime.utcnow().date()
yesterday = (today - timedelta(1))
two_days = (today - timedelta(2))
for stat in ['deleted_cell', 'deleted_wifi']:
p = 0
for i in range(1, 10):
p += i
incr_stat(session, stat, i, two_days)
self.assertEqual(get_curr_stat(session, stat, two_days), p)
self.assertEqual(get_curr_stat(session, stat, yesterday), p)
self.assertEqual(get_curr_stat(session, stat, today), p)
self.assertEqual(get_curr_stat(session, stat), p)
q = p
for i in range(1, 10):
q += i
incr_stat(session, stat, i, yesterday)
self.assertEqual(get_curr_stat(session, stat, two_days), p)
self.assertEqual(get_curr_stat(session, stat, yesterday), q)
self.assertEqual(get_curr_stat(session, stat, today), q)
self.assertEqual(get_curr_stat(session, stat), q)
r = q
for i in range(1, 10):
r += i
incr_stat(session, stat, i, today)
self.assertEqual(get_curr_stat(session, stat, two_days), p)
self.assertEqual(get_curr_stat(session, stat, yesterday), q)
self.assertEqual(get_curr_stat(session, stat, today), r)
self.assertEqual(get_curr_stat(session, stat), r)
