def test_unthrottle_wifi_observations(self):
session = self.db_master_session
block = ObservationBlock()
block.measure_type = ObservationType.wifi
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
k = dict(lat=1.0, lon=1.0)
for i in range(100, 150):
session.add(
WifiObservation(id=i, key=str(i), created=self.really_old))
session.add(Wifi(total_measures=11000, key=str(i), **k))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_wifimeasure_records.delay(batch=7).get()
wifi_unthrottle_measures.delay(10000, 1000).get()
wifis = session.query(Wifi).all()
self.assertEquals(len(wifis), 50)
for wifi in wifis:
if 120 <= int(wifi.key) and int(wifi.key) < 140:
self.assertEquals(wifi.total_measures, 0)
else:
self.assertEquals(wifi.total_measures, 1)
self.check_stats(counter=['items.wifi_unthrottled'])
def test_unthrottle_cell_observations(self):
session = self.db_master_session
block = ObservationBlock()
block.measure_type = ObservationType.cell
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
gsm = RADIO_TYPE['gsm']
k = dict(radio=gsm, mcc=1, mnc=2, lac=4, lat=1.0, lon=1.0)
for i in range(100, 150):
session.add(
CellObservation(id=i, cid=i, created=self.really_old, **k))
session.add(Cell(total_measures=11000, cid=i, **k))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
cell_unthrottle_measures.delay(10000, 1000).get()
cells = session.query(Cell).all()
self.assertEquals(len(cells), 50)
for cell in cells:
if 120 <= cell.cid and cell.cid < 140:
self.assertEquals(cell.total_measures, 0)
else:
self.assertEquals(cell.total_measures, 1)
self.check_stats(counter=['items.cell_unthrottled'])
def test_skip_delete_new_blocks(self):
now = util.utcnow()
today_0000 = now.replace(hour=0, minute=0, second=0, tzinfo=pytz.UTC)
yesterday_0000 = today_0000 - timedelta(days=1)
yesterday_2359 = today_0000 - timedelta(seconds=1)
old = now - timedelta(days=5)
session = self.db_master_session
for i in range(100, 150, 10):
block = ObservationBlock()
block.measure_type = ObservationType.cell
block.start_id = i
block.end_id = i + 10
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
observations = []
for i in range(100, 110):
observations.append(CellObservation(id=i, created=old))
for i in range(110, 120):
observations.append(CellObservation(id=i, created=yesterday_0000))
for i in range(120, 130):
observations.append(CellObservation(id=i, created=yesterday_2359))
for i in range(130, 140):
observations.append(CellObservation(id=i, created=today_0000))
for i in range(140, 150):
observations.append(CellObservation(id=i, created=now))
session.add_all(observations)
session.commit()
def _archived_blocks():
blocks = session.query(ObservationBlock).all()
return len([b for b in blocks if b.archive_date is not None])
def _delete(days=7):
with patch.object(S3Backend,
'check_archive',
lambda x, y, z: True):
delete_cellmeasure_records.delay(days_old=days).get()
session.commit()
_delete(days=7)
self.assertEquals(session.query(CellObservation).count(), 50)
self.assertEqual(_archived_blocks(), 0)
_delete(days=2)
self.assertEquals(session.query(CellObservation).count(), 40)
self.assertEqual(_archived_blocks(), 1)
_delete(days=1)
self.assertEquals(session.query(CellObservation).count(), 20)
self.assertEqual(_archived_blocks(), 3)
_delete(days=0)
self.assertEquals(session.query(CellObservation).count(), 0)
self.assertEqual(_archived_blocks(), 5)
def schedule_observation_archival(self, observation_type,
limit=100, batch=1000000):
blocks = []
obs_meta = OBSERVATION_TYPE_META[observation_type]
obs_cls = obs_meta['class']
with self.db_session() as session:
table_min_id = 0
table_max_id = 0
query = session.query(obs_cls.id).order_by(
obs_cls.id.asc())
record = query.first()
if record is not None:
table_min_id = record[0]
query = session.query(obs_cls.id).order_by(
obs_cls.id.desc())
record = query.first()
if record is not None:
table_max_id = record[0]
if not table_max_id:
# no data in the table
return blocks
query = session.query(ObservationBlock.end_id).filter(
ObservationBlock.measure_type == observation_type).order_by(
ObservationBlock.end_id.desc())
record = query.first()
if record is not None:
min_id = record[0]
else:
min_id = table_min_id
max_id = table_max_id
if max_id - min_id < batch - 1:
# Not enough to fill a block
return blocks
# We're using half-open ranges, so we need to bump the max_id
max_id += 1
this_max_id = min_id + batch
i = 0
while i < limit and (this_max_id - min_id) == batch:
cm_blk = ObservationBlock(start_id=min_id,
end_id=this_max_id,
measure_type=observation_type)
blocks.append((cm_blk.start_id, cm_blk.end_id))
session.add(cm_blk)
min_id = this_max_id
this_max_id = min(batch + this_max_id, max_id)
i += 1
session.commit()
return blocks
def test_delete_wifi_observations(self):
session = self.db_master_session
block = ObservationBlock()
block.measure_type = ObservationType.wifi
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
for i in range(100, 150):
session.add(WifiObservation(id=i, created=self.really_old))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_wifimeasure_records.delay(batch=7).get()
self.assertEquals(session.query(WifiObservation).count(), 30)
self.assertTrue(block.archive_date is not None)