def configure_data(redis_client):
"""
Configure fixed set of data queues.
"""
data_queues = {
# *_incoming need to be the exact same as in webapp.config
'update_incoming': DataQueue('update_incoming', redis_client,
batch=100, compress=True),
'transfer_incoming': DataQueue('transfer_incoming', redis_client,
batch=100, compress=True),
}
for key in ('update_cellarea', ):
data_queues[key] = DataQueue(key, redis_client, batch=100, json=False)
for shard_id in BlueShard.shards().keys():
key = 'update_blue_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in DataMap.shards().keys():
key = 'update_datamap_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500, json=False)
for shard_id in CellShard.shards().keys():
key = 'update_cell_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in WifiShard.shards().keys():
key = 'update_wifi_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
return data_queues
def _queue_and_update_blue(self, obs):
sharded_obs = defaultdict(list)
for ob in obs:
sharded_obs[BlueShard.shard_id(ob.mac)].append(ob)
for shard_id, values in sharded_obs.items():
queue = self.celery_app.data_queues['update_blue_' + shard_id]
queue.enqueue([value.to_json() for value in values])
update_blue.delay(shard_id=shard_id).get()
def _queue_and_update_blue(self, obs):
sharded_obs = defaultdict(list)
for ob in obs:
sharded_obs[BlueShard.shard_id(ob.mac)].append(ob)
for shard_id, values in sharded_obs.items():
queue = self.celery_app.data_queues['update_blue_' + shard_id]
queue.enqueue([value.to_json() for value in values])
update_blue.delay(shard_id=shard_id).get()
def _update_all(self):
update_incoming.delay().get()
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _update_all(self):
schedule_export_reports.delay().get()
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_blue(self, celery, session):
reports = self.add_reports(celery, blue_factor=1, cell_factor=0, wifi_factor=0)
self._update_all(session)
position = reports[0]["position"]
blue_data = reports[0]["bluetoothBeacons"][0]
shard = BlueShard.shard_model(blue_data["macAddress"])
blues = session.query(shard).all()
assert len(blues) == 1
blue = blues[0]
assert blue.lat == pytest.approx(position["latitude"])
assert blue.lon == pytest.approx(position["longitude"])
assert blue.mac == blue_data["macAddress"]
assert blue.samples == 1
def test_blue(self):
reports = self.add_reports(blue_factor=1, cell_factor=0, wifi_factor=0)
self._update_all()
position = reports[0]['position']
wifi_data = reports[0]['bluetoothBeacons'][0]
mac = wifi_data['macAddress']
shard = BlueShard.shard_model(mac)
blues = self.session.query(shard).all()
self.assertEqual(len(blues), 1)
blue = blues[0]
self.assertEqual(blue.lat, position['latitude'])
self.assertEqual(blue.lon, position['longitude'])
self.assertEqual(blue.mac, wifi_data['macAddress'])
self.assertEqual(blue.samples, 1)
def test_blue(self, celery, session):
reports = self.add_reports(
celery, blue_factor=1, cell_factor=0, wifi_factor=0)
self._update_all(session)
position = reports[0]['position']
blue_data = reports[0]['bluetoothBeacons'][0]
shard = BlueShard.shard_model(blue_data['macAddress'])
blues = session.query(shard).all()
assert len(blues) == 1
blue = blues[0]
assert blue.lat == position['latitude']
assert blue.lon == position['longitude']
assert blue.mac == blue_data['macAddress']
assert blue.samples == 1
def test_blue(self):
reports = self.add_reports(blue_factor=1, cell_factor=0, wifi_factor=0)
self._update_all()
position = reports[0]['position']
wifi_data = reports[0]['bluetoothBeacons'][0]
mac = wifi_data['macAddress']
shard = BlueShard.shard_model(mac)
blues = self.session.query(shard).all()
self.assertEqual(len(blues), 1)
blue = blues[0]
self.assertEqual(blue.lat, position['latitude'])
self.assertEqual(blue.lon, position['longitude'])
self.assertEqual(blue.mac, wifi_data['macAddress'])
self.assertEqual(blue.samples, 1)
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name="internal", batch=0, schema="internal")
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name='internal', batch=0, schema='internal')
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_blue(self, celery, session):
reports = self.add_reports(celery,
blue_factor=1,
cell_factor=0,
wifi_factor=0)
self._update_all(session)
position = reports[0]['position']
blue_data = reports[0]['bluetoothBeacons'][0]
shard = BlueShard.shard_model(blue_data['macAddress'])
blues = session.query(shard).all()
assert len(blues) == 1
blue = blues[0]
assert blue.lat == position['latitude']
assert blue.lon == position['longitude']
assert blue.mac == blue_data['macAddress']
assert blue.samples == 1
def test_blue_duplicated(self, celery, session):
self.add_reports(celery, blue_factor=1, cell_factor=0, wifi_factor=0)
# duplicate the Bluetooth entry inside the report
item = self._pop_item(celery)
report = item["report"]
blue = report["bluetoothBeacons"][0]
mac = blue["macAddress"]
report["bluetoothBeacons"].append(blue.copy())
report["bluetoothBeacons"].append(blue.copy())
report["bluetoothBeacons"][1]["signalStrength"] += 2
report["bluetoothBeacons"][2]["signalStrength"] -= 2
self._push_item(celery, item)
self._update_all(session)
shard = BlueShard.shard_model(mac)
blues = session.query(shard).all()
assert len(blues) == 1
assert blues[0].samples == 1
def test_blue_duplicated(self):
self.add_reports(blue_factor=1, cell_factor=0, wifi_factor=0)
# duplicate the Bluetooth entry inside the report
item = self._pop_item()
report = item['report']
blue = report['bluetoothBeacons'][0]
mac = blue['macAddress']
report['bluetoothBeacons'].append(blue.copy())
report['bluetoothBeacons'].append(blue.copy())
report['bluetoothBeacons'][1]['signalStrength'] += 2
report['bluetoothBeacons'][2]['signalStrength'] -= 2
self._push_item(item)
self._update_all()
shard = BlueShard.shard_model(mac)
blues = self.session.query(shard).all()
self.assertEqual(len(blues), 1)
self.assertEqual(blues[0].samples, 1)
def test_blue_duplicated(self, celery, session):
self.add_reports(
celery, blue_factor=1, cell_factor=0, wifi_factor=0)
# duplicate the Bluetooth entry inside the report
item = self._pop_item(celery)
report = item['report']
blue = report['bluetoothBeacons'][0]
mac = blue['macAddress']
report['bluetoothBeacons'].append(blue.copy())
report['bluetoothBeacons'].append(blue.copy())
report['bluetoothBeacons'][1]['signalStrength'] += 2
report['bluetoothBeacons'][2]['signalStrength'] -= 2
self._push_item(celery, item)
self._update_all(session)
shard = BlueShard.shard_model(mac)
blues = session.query(shard).all()
assert len(blues) == 1
assert blues[0].samples == 1
def test_blue_duplicated(self):
self.add_reports(blue_factor=1, cell_factor=0, wifi_factor=0)
# duplicate the Bluetooth entry inside the report
queue = self.celery_app.export_queues['queue_export_internal']
items = queue.dequeue(queue.queue_key())
report = items[0]['report']
blue = report['bluetoothBeacons'][0]
mac = blue['macAddress']
report['bluetoothBeacons'].append(blue.copy())
report['bluetoothBeacons'].append(blue.copy())
report['bluetoothBeacons'][1]['signalStrength'] += 2
report['bluetoothBeacons'][2]['signalStrength'] -= 2
queue.enqueue(items, queue.queue_key())
self._update_all()
shard = BlueShard.shard_model(mac)
blues = self.session.query(shard).all()
self.assertEqual(len(blues), 1)
self.assertEqual(blues[0].samples, 1)
def test_new(self):
utcnow = util.utcnow()
obs = BlueObservationFactory.build()
self._queue_and_update_blue([obs])
shard = BlueShard.shard_model(obs.mac)
blues = self.session.query(shard).all()
self.assertEqual(len(blues), 1)
blue = blues[0]
self.assertAlmostEqual(blue.lat, obs.lat)
self.assertAlmostEqual(blue.max_lat, obs.lat)
self.assertAlmostEqual(blue.min_lat, obs.lat)
self.assertAlmostEqual(blue.lon, obs.lon)
self.assertAlmostEqual(blue.max_lon, obs.lon)
self.assertAlmostEqual(blue.min_lon, obs.lon)
self.assertEqual(blue.radius, 0)
self.assertEqual(blue.region, 'GB')
self.assertEqual(blue.samples, 1)
self.assertAlmostEqual(blue.weight, 1.0, 2)
self.assertEqual(blue.created.date(), utcnow.date())
self.assertEqual(blue.modified.date(), utcnow.date())
self.assertEqual(blue.block_first, None)
self.assertEqual(blue.block_last, None)
self.assertEqual(blue.block_count, None)
def test_new(self):
utcnow = util.utcnow()
obs = BlueObservationFactory.build()
self._queue_and_update_blue([obs])
shard = BlueShard.shard_model(obs.mac)
blues = self.session.query(shard).all()
self.assertEqual(len(blues), 1)
blue = blues[0]
self.assertAlmostEqual(blue.lat, obs.lat)
self.assertAlmostEqual(blue.max_lat, obs.lat)
self.assertAlmostEqual(blue.min_lat, obs.lat)
self.assertAlmostEqual(blue.lon, obs.lon)
self.assertAlmostEqual(blue.max_lon, obs.lon)
self.assertAlmostEqual(blue.min_lon, obs.lon)
self.assertEqual(blue.radius, 0)
self.assertEqual(blue.region, 'GB')
self.assertEqual(blue.samples, 1)
self.assertAlmostEqual(blue.weight, 1.0, 2)
self.assertEqual(blue.created.date(), utcnow.date())
self.assertEqual(blue.modified.date(), utcnow.date())
self.assertEqual(blue.block_first, None)
self.assertEqual(blue.block_last, None)
self.assertEqual(blue.block_count, None)
def configure_data(redis_client):
"""
Configure fixed set of data queues.
"""
data_queues = {
# update_incoming needs to be the exact same as in webapp.config
'update_incoming': DataQueue('update_incoming', redis_client,
batch=100, compress=True),
}
for key in ('update_cellarea', 'update_cellarea_ocid'):
data_queues[key] = DataQueue(key, redis_client, batch=100, json=False)
for shard_id in BlueShard.shards().keys():
key = 'update_blue_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in DataMap.shards().keys():
key = 'update_datamap_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500, json=False)
for shard_id in CellShard.shards().keys():
key = 'update_cell_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
for shard_id in WifiShard.shards().keys():
key = 'update_wifi_' + shard_id
data_queues[key] = DataQueue(key, redis_client, batch=500)
return data_queues
