def _queue_and_update(self, obs):
sharded_obs = defaultdict(list)
for ob in obs:
sharded_obs[WifiShard.shard_id(ob.mac)].append(ob)
for shard_id, values in sharded_obs.items():
queue = self.celery_app.data_queues['update_wifi_' + shard_id]
queue.enqueue(values)
update_wifi.delay(shard_id=shard_id).get()
def _queue_and_update(self, obs):
sharded_obs = defaultdict(list)
for ob in obs:
sharded_obs[WifiShard.shard_id(ob.mac)].append(ob)
for shard_id, values in sharded_obs.items():
queue = self.celery_app.data_queues['update_wifi_' + shard_id]
queue.enqueue(values)
update_wifi.delay(shard_id=shard_id).get()
def process_reports(self, reports, userid=None):
malformed_reports = 0
positions = set()
observations = {'cell': [], 'wifi': []}
obs_count = {
'cell': {'upload': 0, 'drop': 0},
'wifi': {'upload': 0, 'drop': 0},
}
new_station_count = {'cell': 0, 'wifi': 0}
for report in reports:
cell, wifi, malformed_obs = self.process_report(report)
if cell:
observations['cell'].extend(cell)
obs_count['cell']['upload'] += len(cell)
if wifi:
observations['wifi'].extend(wifi)
obs_count['wifi']['upload'] += len(wifi)
if (cell or wifi):
positions.add((report['lat'], report['lon']))
else:
malformed_reports += 1
for name in ('cell', 'wifi'):
obs_count[name]['drop'] += malformed_obs[name]
# group by unique station key
for name in ('cell', 'wifi'):
station_keys = set()
for obs in observations[name]:
if name == 'cell':
station_keys.add(obs.cellid)
elif name == 'wifi':
station_keys.add(obs.mac)
# determine scores for stations
new_station_count[name] += self.new_stations(name, station_keys)
if observations['cell']:
sharded_obs = defaultdict(list)
for ob in observations['cell']:
shard_id = CellShard.shard_id(ob.cellid)
sharded_obs[shard_id].append(ob)
for shard_id, values in sharded_obs.items():
cell_queue = self.data_queues['update_cell_' + shard_id]
cell_queue.enqueue(list(values), pipe=self.pipe)
if observations['wifi']:
sharded_obs = defaultdict(list)
for ob in observations['wifi']:
shard_id = WifiShard.shard_id(ob.mac)
sharded_obs[shard_id].append(ob)
for shard_id, values in sharded_obs.items():
wifi_queue = self.data_queues['update_wifi_' + shard_id]
wifi_queue.enqueue(list(values), pipe=self.pipe)
self.process_datamap(positions)
self.process_score(userid, positions, new_station_count)
self.emit_stats(
len(reports),
malformed_reports,
obs_count,
)
def process_reports(self, reports, userid=None):
malformed_reports = 0
positions = set()
observations = {'cell': [], 'wifi': []}
obs_count = {
'cell': {
'upload': 0,
'drop': 0
},
'wifi': {
'upload': 0,
'drop': 0
},
}
new_station_count = {'cell': 0, 'wifi': 0}
for report in reports:
cell, wifi, malformed_obs = self.process_report(report)
if cell:
observations['cell'].extend(cell)
obs_count['cell']['upload'] += len(cell)
if wifi:
observations['wifi'].extend(wifi)
obs_count['wifi']['upload'] += len(wifi)
if (cell or wifi):
positions.add((report['lat'], report['lon']))
else:
malformed_reports += 1
for name in ('cell', 'wifi'):
obs_count[name]['drop'] += malformed_obs[name]
# group by unique station key
for name in ('cell', 'wifi'):
station_keys = set()
for obs in observations[name]:
if name == 'cell':
station_keys.add(obs.cellid)
elif name == 'wifi':
station_keys.add(obs.mac)
# determine scores for stations
new_station_count[name] += self.new_stations(name, station_keys)
if observations['cell']:
sharded_obs = defaultdict(list)
for ob in observations['cell']:
shard_id = CellShard.shard_id(ob.cellid)
sharded_obs[shard_id].append(ob)
for shard_id, values in sharded_obs.items():
cell_queue = self.data_queues['update_cell_' + shard_id]
cell_queue.enqueue(list(values), pipe=self.pipe)
if observations['wifi']:
sharded_obs = defaultdict(list)
for ob in observations['wifi']:
shard_id = WifiShard.shard_id(ob.mac)
sharded_obs[shard_id].append(ob)
for shard_id, values in sharded_obs.items():
wifi_queue = self.data_queues['update_wifi_' + shard_id]
wifi_queue.enqueue(list(values), pipe=self.pipe)
self.process_datamap(positions)
self.process_score(userid, positions, new_station_count)
self.emit_stats(
len(reports),
malformed_reports,
obs_count,
)
