def _queue_and_update(self, obs): sharded_obs = defaultdict(list) for ob in obs: sharded_obs[WifiShard.shard_id(ob.mac)].append(ob) for shard_id, values in sharded_obs.items(): queue = self.celery_app.data_queues['update_wifi_' + shard_id] queue.enqueue(values) update_wifi.delay(shard_id=shard_id).get()
def process_reports(self, reports, userid=None): malformed_reports = 0 positions = set() observations = {'cell': [], 'wifi': []} obs_count = { 'cell': {'upload': 0, 'drop': 0}, 'wifi': {'upload': 0, 'drop': 0}, } new_station_count = {'cell': 0, 'wifi': 0} for report in reports: cell, wifi, malformed_obs = self.process_report(report) if cell: observations['cell'].extend(cell) obs_count['cell']['upload'] += len(cell) if wifi: observations['wifi'].extend(wifi) obs_count['wifi']['upload'] += len(wifi) if (cell or wifi): positions.add((report['lat'], report['lon'])) else: malformed_reports += 1 for name in ('cell', 'wifi'): obs_count[name]['drop'] += malformed_obs[name] # group by unique station key for name in ('cell', 'wifi'): station_keys = set() for obs in observations[name]: if name == 'cell': station_keys.add(obs.cellid) elif name == 'wifi': station_keys.add(obs.mac) # determine scores for stations new_station_count[name] += self.new_stations(name, station_keys) if observations['cell']: sharded_obs = defaultdict(list) for ob in observations['cell']: shard_id = CellShard.shard_id(ob.cellid) sharded_obs[shard_id].append(ob) for shard_id, values in sharded_obs.items(): cell_queue = self.data_queues['update_cell_' + shard_id] cell_queue.enqueue(list(values), pipe=self.pipe) if observations['wifi']: sharded_obs = defaultdict(list) for ob in observations['wifi']: shard_id = WifiShard.shard_id(ob.mac) sharded_obs[shard_id].append(ob) for shard_id, values in sharded_obs.items(): wifi_queue = self.data_queues['update_wifi_' + shard_id] wifi_queue.enqueue(list(values), pipe=self.pipe) self.process_datamap(positions) self.process_score(userid, positions, new_station_count) self.emit_stats( len(reports), malformed_reports, obs_count, )
def process_reports(self, reports, userid=None): malformed_reports = 0 positions = set() observations = {'cell': [], 'wifi': []} obs_count = { 'cell': { 'upload': 0, 'drop': 0 }, 'wifi': { 'upload': 0, 'drop': 0 }, } new_station_count = {'cell': 0, 'wifi': 0} for report in reports: cell, wifi, malformed_obs = self.process_report(report) if cell: observations['cell'].extend(cell) obs_count['cell']['upload'] += len(cell) if wifi: observations['wifi'].extend(wifi) obs_count['wifi']['upload'] += len(wifi) if (cell or wifi): positions.add((report['lat'], report['lon'])) else: malformed_reports += 1 for name in ('cell', 'wifi'): obs_count[name]['drop'] += malformed_obs[name] # group by unique station key for name in ('cell', 'wifi'): station_keys = set() for obs in observations[name]: if name == 'cell': station_keys.add(obs.cellid) elif name == 'wifi': station_keys.add(obs.mac) # determine scores for stations new_station_count[name] += self.new_stations(name, station_keys) if observations['cell']: sharded_obs = defaultdict(list) for ob in observations['cell']: shard_id = CellShard.shard_id(ob.cellid) sharded_obs[shard_id].append(ob) for shard_id, values in sharded_obs.items(): cell_queue = self.data_queues['update_cell_' + shard_id] cell_queue.enqueue(list(values), pipe=self.pipe) if observations['wifi']: sharded_obs = defaultdict(list) for ob in observations['wifi']: shard_id = WifiShard.shard_id(ob.mac) sharded_obs[shard_id].append(ob) for shard_id, values in sharded_obs.items(): wifi_queue = self.data_queues['update_wifi_' + shard_id] wifi_queue.enqueue(list(values), pipe=self.pipe) self.process_datamap(positions) self.process_score(userid, positions, new_station_count) self.emit_stats( len(reports), malformed_reports, obs_count, )