def test_unthrottle_wifi_measures(self):
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE_CODE['wifi']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
k = dict(lat=1.0, lon=1.0)
for i in range(100, 150):
session.add(WifiMeasure(id=i, key=str(i), created=self.really_old))
session.add(Wifi(total_measures=11000, key=str(i), **k))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_wifimeasure_records.delay(batch=7).get()
wifi_unthrottle_measures.delay(10000, 1000).get()
wifis = session.query(Wifi).all()
self.assertEquals(len(wifis), 50)
for wifi in wifis:
if 120 <= int(wifi.key) and int(wifi.key) < 140:
self.assertEquals(wifi.total_measures, 0)
else:
self.assertEquals(wifi.total_measures, 1)
self.check_stats(counter=['items.wifi_unthrottled'])
def test_unthrottle_cell_measures(self):
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE_CODE['cell']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
k = dict(mcc=1, mnc=2, lac=4, lat=1.0, lon=1.0)
for i in range(100, 150):
session.add(CellMeasure(id=i, cid=i, created=self.really_old, **k))
session.add(Cell(total_measures=11000, cid=i, **k))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
cell_unthrottle_measures.delay(10000, 1000).get()
cells = session.query(Cell).all()
self.assertEquals(len(cells), 50)
for cell in cells:
if 120 <= cell.cid and cell.cid < 140:
self.assertEquals(cell.total_measures, 0)
else:
self.assertEquals(cell.total_measures, 1)
self.check_stats(counter=['items.cell_unthrottled'])
def schedule_measure_archival(self, measure_type, limit=100, batch=1000000):
blocks = []
measure_meta = MEASURE_TYPE_META[measure_type]
measure_cls = measure_meta['class']
with self.db_session() as session:
table_min_id = 0
table_max_id = 0
query = session.query(measure_cls.id).order_by(
measure_cls.id.asc())
record = query.first()
if record is not None:
table_min_id = record[0]
query = session.query(measure_cls.id).order_by(
measure_cls.id.desc())
record = query.first()
if record is not None:
table_max_id = record[0]
if not table_max_id:
# no data in the table
return blocks
query = session.query(MeasureBlock.end_id).filter(
MeasureBlock.measure_type == measure_type).order_by(
MeasureBlock.end_id.desc())
record = query.first()
if record is not None:
min_id = record[0]
else:
min_id = table_min_id
max_id = table_max_id
if max_id - min_id < batch - 1:
# Not enough to fill a block
return blocks
# We're using half-open ranges, so we need to bump the max_id
max_id += 1
this_max_id = min_id + batch
i = 0
while i < limit and (this_max_id - min_id) == batch:
cm_blk = MeasureBlock(start_id=min_id,
end_id=this_max_id,
measure_type=measure_type)
blocks.append((cm_blk.start_id, cm_blk.end_id))
session.add(cm_blk)
min_id = this_max_id
this_max_id = min(batch + this_max_id, max_id)
i += 1
session.commit()
return blocks
def test_delete_wifi_measures(self):
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE_CODE['wifi']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
for i in range(100, 150):
session.add(WifiMeasure(id=i, created=self.really_old))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_wifimeasure_records.delay(batch=7).get()
self.assertEquals(session.query(WifiMeasure).count(), 30)
self.assertTrue(block.archive_date is not None)
def test_skip_delete_new_blocks(self):
now = util.utcnow()
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE_CODE['cell']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
measures = []
for i in range(100, 150):
measures.append(CellMeasure(id=i, created=now))
session.add_all(measures)
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
self.assertEquals(session.query(CellMeasure).count(), 50)
# The archive_date should *not* be set as we haven't deleted
# the actual records yet
self.assertTrue(block.archive_date is None)
# Update all the create dates to be far in the past
for m in measures:
m.created = self.really_old
session.add_all(measures)
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
self.assertEquals(session.query(CellMeasure).count(), 30)
# The archive_date should now be set as the measure records
# have been deleted.
self.assertTrue(block.archive_date is not None)
def test_delete_wifi_measures(self):
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE['wifi']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
for i in range(100, 150):
session.add(WifiMeasure(id=i))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_wifimeasure_records()
self.assertEquals(session.query(WifiMeasure).count(), 29)
self.assertTrue(block.archive_date is not None)
def test_delete_cell_measures(self):
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE_CODE['cell']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
for i in range(100, 150):
session.add(CellMeasure(id=i, created=self.really_old))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
self.assertEquals(session.query(CellMeasure).count(), 30)
self.assertTrue(block.archive_date is not None)
def test_skip_delete_new_blocks(self):
now = util.utcnow()
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE_CODE['cell']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
measures = []
for i in range(100, 150):
measures.append(CellMeasure(id=i, created=now))
session.add_all(measures)
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
self.assertEquals(session.query(CellMeasure).count(), 50)
# The archive_date should *not* be set as we haven't deleted
# the actual records yet
self.assertTrue(block.archive_date is None)
# Update all the create dates to be far in the past
for m in measures:
m.created = self.really_old
session.add_all(measures)
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
self.assertEquals(session.query(CellMeasure).count(), 30)
# The archive_date should now be set as the measure records
# have been deleted.
self.assertTrue(block.archive_date is not None)