def test_position_invalid(self, celery, session, metricsmock):
self.add_reports(celery,
1,
cell_factor=0,
wifi_factor=1,
wifi_key="000000123456",
lat=-90.1)
self.add_reports(celery,
1,
cell_factor=0,
wifi_factor=1,
wifi_key="000000234567")
self._update_all(session)
shard = WifiShard.shards()["0"]
assert session.query(shard).count() == 1
metricsmock.assert_incr_once("data.report.upload",
value=2,
tags=["key:test"])
metricsmock.assert_incr_once("data.report.drop",
value=1,
tags=["key:test"])
metricsmock.assert_incr_once("data.observation.insert",
value=1,
tags=["type:wifi"])
metricsmock.assert_incr_once("data.observation.upload",
tags=["type:wifi", "key:test"])
def configure_data(redis_client):
"""
Configure fixed set of data queues.
"""
data_queues = {
'update_cell': DataQueue('update_cell', redis_client,
queue_key='update_cell'), # BBB
'update_cellarea': DataQueue('update_cellarea', redis_client,
queue_key='update_cellarea'),
'update_cellarea_ocid': DataQueue('update_cellarea_ocid', redis_client,
queue_key='update_cellarea_ocid'),
'update_score': DataQueue('update_score', redis_client,
queue_key='update_score'),
}
for shard_id in DataMap.shards().keys():
name = 'update_datamap_' + shard_id
data_queues[name] = DataQueue(name, redis_client, queue_key=name)
for shard_id in CellShard.shards().keys():
name = 'update_cell_' + shard_id
data_queues[name] = DataQueue(
name, redis_client, queue_key=name)
for shard_id in WifiShard.shards().keys():
name = 'update_wifi_' + shard_id
data_queues[name] = DataQueue(
name, redis_client, queue_key=name)
return data_queues
def configure_data(redis_client):
"""
Configure fixed set of data queues.
"""
data_queues = {
'update_cell':
DataQueue('update_cell', redis_client, queue_key='update_cell'), # BBB
'update_cellarea':
DataQueue('update_cellarea', redis_client,
queue_key='update_cellarea'),
'update_cellarea_ocid':
DataQueue('update_cellarea_ocid',
redis_client,
queue_key='update_cellarea_ocid'),
'update_score':
DataQueue('update_score', redis_client, queue_key='update_score'),
}
for shard_id in DataMap.shards().keys():
name = 'update_datamap_' + shard_id
data_queues[name] = DataQueue(name, redis_client, queue_key=name)
for shard_id in CellShard.shards().keys():
name = 'update_cell_' + shard_id
data_queues[name] = DataQueue(name, redis_client, queue_key=name)
for shard_id in WifiShard.shards().keys():
name = 'update_wifi_' + shard_id
data_queues[name] = DataQueue(name, redis_client, queue_key=name)
return data_queues
def configure_data(redis_client):
"""
Configure fixed set of data queues.
"""
data_queues = {
# *_incoming need to be the exact same as in webapp.config
'update_incoming': DataQueue('update_incoming', redis_client,
batch=100, compress=True),
'transfer_incoming': DataQueue('transfer_incoming', redis_client,
batch=100, compress=True),
}
for key in ('update_cellarea', ):
data_queues[key] = DataQueue(key, redis_client, batch=100, json=False)
for shard_id in BlueShard.shards().keys():
key = 'update_blue_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in DataMap.shards().keys():
key = 'update_datamap_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500, json=False)
for shard_id in CellShard.shards().keys():
key = 'update_cell_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in WifiShard.shards().keys():
key = 'update_wifi_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
return data_queues
def _update_all(self):
schedule_export_reports.delay().get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _update_all(self):
schedule_export_reports.delay().get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _update_all(self):
update_incoming.delay().get()
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def __call__(self):
cells = (self.session.query(CellArea.region, CellArea.radio,
func.sum(CellArea.num_cells)).filter(
CellArea.region.isnot(None)).group_by(
CellArea.region,
CellArea.radio)).all()
default = {'gsm': 0, 'wcdma': 0, 'lte': 0, 'wifi': 0}
stats = {}
for region, radio, num in cells:
if region not in stats:
stats[region] = default.copy()
stats[region][radio.name] = int(num)
for shard in WifiShard.shards().values():
wifis = (self.session.query(shard.region, func.count()).filter(
shard.region.isnot(None)).group_by(shard.region)).all()
for region, num in wifis:
if region not in stats:
stats[region] = default.copy()
stats[region]['wifi'] += int(num)
if not stats:
return
region_stats = dict(
self.session.query(RegionStat.region, RegionStat).all())
for region, values in stats.items():
if region in region_stats:
region_stats[region].gsm = values['gsm']
region_stats[region].wcdma = values['wcdma']
region_stats[region].lte = values['lte']
region_stats[region].wifi = values['wifi']
else:
self.session.add(
RegionStat(
region=region,
gsm=values['gsm'],
wcdma=values['wcdma'],
lte=values['lte'],
wifi=values['wifi'],
))
obsolete_regions = list(set(region_stats.keys()) - set(stats.keys()))
if obsolete_regions:
(self.session.query(RegionStat).filter(
RegionStat.region.in_(obsolete_regions))).delete(
synchronize_session=False)
def __call__(self):
cells = (self.session.query(CellArea.region,
CellArea.radio,
func.sum(CellArea.num_cells))
.filter(CellArea.region.isnot(None))
.group_by(CellArea.region, CellArea.radio)).all()
default = {'gsm': 0, 'wcdma': 0, 'lte': 0, 'wifi': 0}
stats = {}
for region, radio, num in cells:
if region not in stats:
stats[region] = default.copy()
stats[region][radio.name] = int(num)
for shard in WifiShard.shards().values():
wifis = (self.session.query(shard.region, func.count())
.filter(shard.region.isnot(None))
.group_by(shard.region)).all()
for region, num in wifis:
if region not in stats:
stats[region] = default.copy()
stats[region]['wifi'] += int(num)
if not stats:
return
region_stats = dict(self.session.query(RegionStat.region,
RegionStat).all())
for region, values in stats.items():
if region in region_stats:
region_stats[region].gsm = values['gsm']
region_stats[region].wcdma = values['wcdma']
region_stats[region].lte = values['lte']
region_stats[region].wifi = values['wifi']
else:
self.session.add(RegionStat(
region=region,
gsm=values['gsm'],
wcdma=values['wcdma'],
lte=values['lte'],
wifi=values['wifi'],
))
obsolete_regions = list(set(region_stats.keys()) - set(stats.keys()))
if obsolete_regions:
(self.session.query(RegionStat)
.filter(RegionStat.region.in_(obsolete_regions))
).delete(synchronize_session=False)
def test_position_invalid(self):
self.add_reports(1, cell_factor=0, wifi_factor=1,
wifi_key='000000123456', lat=-90.1)
self.add_reports(1, cell_factor=0, wifi_factor=1,
wifi_key='000000234567')
self._update_all()
shard = WifiShard.shards()['0']
self.assertEqual(self.session.query(shard).count(), 1)
self.check_stats(counter=[
('data.report.upload', 1, 2, ['key:test']),
('data.report.drop', 1, 1, ['reason:malformed', 'key:test']),
('data.observation.insert', 1, 1, ['type:wifi']),
('data.observation.upload', 1, 1, ['type:wifi', 'key:test']),
])
def test_position_invalid(self):
self.add_reports(1, cell_factor=0, wifi_factor=1,
wifi_key='000000123456', lat=-90.1)
self.add_reports(1, cell_factor=0, wifi_factor=1,
wifi_key='000000234567')
self._update_all()
shard = WifiShard.shards()['0']
self.assertEqual(self.session.query(shard).count(), 1)
self.check_stats(counter=[
('data.report.upload', 1, 2, ['key:test']),
('data.report.drop', 1, 1, ['key:test']),
('data.observation.insert', 1, 1, ['type:wifi']),
('data.observation.upload', 1, 1, ['type:wifi', 'key:test']),
])
def test_position_invalid(self, celery, session, stats):
self.add_reports(celery, 1, cell_factor=0, wifi_factor=1,
wifi_key='000000123456', lat=-90.1)
self.add_reports(celery, 1, cell_factor=0, wifi_factor=1,
wifi_key='000000234567')
self._update_all(session)
shard = WifiShard.shards()['0']
assert session.query(shard).count() == 1
stats.check(counter=[
('data.report.upload', 1, 2, ['key:test']),
('data.report.drop', 1, 1, ['key:test']),
('data.observation.insert', 1, 1, ['type:wifi']),
('data.observation.upload', 1, 1, ['type:wifi', 'key:test']),
])
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name="internal", batch=0, schema="internal")
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name='internal', batch=0, schema='internal')
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def configure_data(redis_client):
"""
Configure fixed set of data queues.
"""
data_queues = {
# update_incoming needs to be the exact same as in webapp.config
'update_incoming': DataQueue('update_incoming', redis_client,
batch=100, compress=True),
}
for key in ('update_cellarea', 'update_cellarea_ocid'):
data_queues[key] = DataQueue(key, redis_client, batch=100, json=False)
for shard_id in BlueShard.shards().keys():
key = 'update_blue_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in DataMap.shards().keys():
key = 'update_datamap_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500, json=False)
for shard_id in CellShard.shards().keys():
key = 'update_cell_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in WifiShard.shards().keys():
key = 'update_wifi_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
return data_queues
def celerybeat_schedule(app_config):
"""Return the celery beat schedule as a dictionary."""
sections = app_config.sections()
schedule = {
# Monitoring
'monitor-queue-size': {
'task': 'ichnaea.data.tasks.monitor_queue_size',
'schedule': timedelta(seconds=60),
'options': {
'expires': 57
},
},
'monitor-api-users': {
'task': 'ichnaea.data.tasks.monitor_api_users',
'schedule': timedelta(seconds=600),
'options': {
'expires': 570
},
},
'monitor-api-key-limits': {
'task': 'ichnaea.data.tasks.monitor_api_key_limits',
'schedule': timedelta(seconds=600),
'options': {
'expires': 570
},
},
# Statistics
'update-statcounter': {
'task': 'ichnaea.data.tasks.update_statcounter',
'args': (1, ),
'schedule': crontab(minute=3),
'options': {
'expires': 2700
},
},
'update-statregion': {
'task': 'ichnaea.data.tasks.update_statregion',
'schedule': crontab(minute=5),
'options': {
'expires': 2700
},
},
# Data Pipeline
'schedule-export-reports': {
'task': 'ichnaea.data.tasks.schedule_export_reports',
'schedule': timedelta(seconds=8),
'options': {
'expires': 15
},
},
'update-cellarea': {
'task': 'ichnaea.data.tasks.update_cellarea',
'schedule': timedelta(seconds=8),
'args': (100, ),
'options': {
'expires': 15
},
},
'update-cellarea-ocid': {
'task': 'ichnaea.data.tasks.update_cellarea_ocid',
'schedule': timedelta(seconds=9),
'args': (100, ),
'options': {
'expires': 15
},
},
'update-score': {
'task': 'ichnaea.data.tasks.update_score',
'args': (250, ),
'schedule': timedelta(seconds=9),
'options': {
'expires': 10
},
},
}
for shard_id in CellShard.shards().keys():
schedule.update({
'update-cell-' + shard_id: {
'task': 'ichnaea.data.tasks.update_cell',
'schedule': timedelta(seconds=7),
'args': (500, shard_id),
'options': {
'expires': 10
},
}
})
for shard_id in DataMap.shards().keys():
schedule.update({
'update-datamap-' + shard_id: {
'task': 'ichnaea.data.tasks.update_datamap',
'args': (500, shard_id),
'schedule': timedelta(seconds=14),
'options': {
'expires': 20
},
},
})
for shard_id in WifiShard.shards().keys():
schedule.update({
'update-wifi-' + shard_id: {
'task': 'ichnaea.data.tasks.update_wifi',
'schedule': timedelta(seconds=6),
'args': (500, shard_id),
'options': {
'expires': 10
},
}
})
if 'assets' in sections and app_config.get('assets', 'bucket', None):
# only configure tasks if target bucket is configured
schedule.update({
'cell-export-full': {
'task': 'ichnaea.data.tasks.cell_export_full',
'schedule': crontab(hour=0, minute=13),
'options': {
'expires': 39600
},
},
'cell-export-diff': {
'task': 'ichnaea.data.tasks.cell_export_diff',
'schedule': crontab(minute=3),
'options': {
'expires': 2700
},
},
})
if 'import:ocid' in sections:
schedule.update({
'monitor-ocid-import': {
'task': 'ichnaea.data.tasks.monitor_ocid_import',
'schedule': timedelta(seconds=600),
'options': {
'expires': 570
},
},
'cell-import-external': {
'task': 'ichnaea.data.tasks.cell_import_external',
'args': (True, ),
'schedule': crontab(minute=52),
'options': {
'expires': 2700
},
},
})
return schedule
def celerybeat_schedule(app_config):
"""Return the celery beat schedule as a dictionary."""
sections = app_config.sections()
schedule = {
# Monitoring
'monitor-queue-size': {
'task': 'ichnaea.data.tasks.monitor_queue_size',
'schedule': timedelta(seconds=60),
'options': {'expires': 57},
},
'monitor-api-users': {
'task': 'ichnaea.data.tasks.monitor_api_users',
'schedule': timedelta(seconds=600),
'options': {'expires': 570},
},
'monitor-api-key-limits': {
'task': 'ichnaea.data.tasks.monitor_api_key_limits',
'schedule': timedelta(seconds=600),
'options': {'expires': 570},
},
# Statistics
'update-statcounter': {
'task': 'ichnaea.data.tasks.update_statcounter',
'args': (1, ),
'schedule': crontab(minute=3),
'options': {'expires': 2700},
},
'update-statregion': {
'task': 'ichnaea.data.tasks.update_statregion',
'schedule': timedelta(seconds=3600 * 6),
'options': {'expires': 3600 * 5},
},
# Data Pipeline
'schedule-export-reports': {
'task': 'ichnaea.data.tasks.schedule_export_reports',
'schedule': timedelta(seconds=8),
'options': {'expires': 15},
},
'update-cellarea': {
'task': 'ichnaea.data.tasks.update_cellarea',
'schedule': timedelta(seconds=8),
'args': (100, ),
'options': {'expires': 15},
},
'update-cellarea-ocid': {
'task': 'ichnaea.data.tasks.update_cellarea_ocid',
'schedule': timedelta(seconds=9),
'args': (100, ),
'options': {'expires': 15},
},
'update-score': {
'task': 'ichnaea.data.tasks.update_score',
'args': (250, ),
'schedule': timedelta(seconds=9),
'options': {'expires': 10},
},
}
for shard_id in CellShard.shards().keys():
schedule.update({
'update-cell-' + shard_id: {
'task': 'ichnaea.data.tasks.update_cell',
'schedule': timedelta(seconds=7),
'args': (500, shard_id),
'options': {'expires': 10},
}
})
for shard_id in DataMap.shards().keys():
schedule.update({
'update-datamap-' + shard_id: {
'task': 'ichnaea.data.tasks.update_datamap',
'args': (500, shard_id),
'schedule': timedelta(seconds=14),
'options': {'expires': 20},
},
})
for shard_id in WifiShard.shards().keys():
schedule.update({
'update-wifi-' + shard_id: {
'task': 'ichnaea.data.tasks.update_wifi',
'schedule': timedelta(seconds=6),
'args': (500, shard_id),
'options': {'expires': 10},
}
})
if 'assets' in sections and app_config.get('assets', 'bucket', None):
# only configure tasks if target bucket is configured
schedule.update({
'cell-export-full': {
'task': 'ichnaea.data.tasks.cell_export_full',
'schedule': crontab(hour=0, minute=13),
'options': {'expires': 39600},
},
'cell-export-diff': {
'task': 'ichnaea.data.tasks.cell_export_diff',
'schedule': crontab(minute=3),
'options': {'expires': 2700},
},
})
if 'import:ocid' in sections:
schedule.update({
'monitor-ocid-import': {
'task': 'ichnaea.data.tasks.monitor_ocid_import',
'schedule': timedelta(seconds=600),
'options': {'expires': 570},
},
'cell-import-external': {
'task': 'ichnaea.data.tasks.cell_import_external',
'args': (True, ),
'schedule': crontab(minute=52),
'options': {'expires': 2700},
},
})
return schedule
