def query_cells(query, lookups, model, raven_client): # Given a location query and a list of lookup instances, query the # database and return a list of model objects. cellids = [lookup.cellid for lookup in lookups] if not cellids: # pragma: no cover return [] # load all fields used in score calculation and those we # need for the position load_fields = ('cellid', 'lat', 'lon', 'radius', 'region', 'samples', 'created', 'modified', 'last_seen', 'block_last', 'block_count') result = [] today = util.utcnow().date() try: shards = defaultdict(list) for lookup in lookups: shards[model.shard_model(lookup.radioType)].append(lookup.cellid) for shard, shard_cellids in shards.items(): columns = shard.__table__.c fields = [getattr(columns, f) for f in load_fields] rows = (query.session.execute( select(fields).where(columns.lat.isnot(None)).where( columns.lon.isnot(None)).where( columns.cellid.in_(shard_cellids)))).fetchall() result.extend( [row for row in rows if not station_blocked(row, today)]) except Exception: raven_client.captureException() return result
def query_macs(query, lookups, raven_client, db_model): macs = [lookup.mac for lookup in lookups] if not macs: # pragma: no cover return [] # load all fields used in score calculation and those we # need for the position or region load_fields = ('mac', 'lat', 'lon', 'radius', 'region', 'samples', 'created', 'modified', 'last_seen', 'block_last', 'block_count') result = [] today = util.utcnow().date() try: shards = defaultdict(list) for mac in macs: shards[db_model.shard_model(mac)].append(mac) for shard, shard_macs in shards.items(): columns = shard.__table__.c fields = [getattr(columns, f) for f in load_fields] rows = (query.session.execute( select(fields).where(columns.lat.isnot(None)).where( columns.lon.isnot(None)).where( columns.mac.in_(shard_macs)))).fetchall() result.extend( [row for row in rows if not station_blocked(row, today)]) except Exception: raven_client.captureException() return result
def query_macs(query, lookups, raven_client, db_model): macs = [lookup.mac for lookup in lookups] if not macs: # pragma: no cover return [] # load all fields used in score calculation and those we # need for the position or region load_fields = ('mac', 'lat', 'lon', 'radius', 'region', 'samples', 'created', 'modified', 'last_seen', 'block_last', 'block_count') result = [] today = util.utcnow().date() try: shards = defaultdict(list) for mac in macs: shards[db_model.shard_model(mac)].append(mac) for shard, shard_macs in shards.items(): columns = shard.__table__.c fields = [getattr(columns, f) for f in load_fields] rows = ( query.session.execute( select(fields) .where(columns.lat.isnot(None)) .where(columns.lon.isnot(None)) .where(columns.mac.in_(shard_macs))) ).fetchall() result.extend([row for row in rows if not station_blocked(row, today)]) except Exception: raven_client.captureException() return result
def query_stations(self, session, shard, shard_values): blocklist = {} stations = {} keys = list(shard_values.keys()) rows = self.query_shard(session, shard, keys) for row in rows: unique_key = row.unique_key stations[unique_key] = row blocklist[unique_key] = station_blocked(row, self.today) return (blocklist, stations)
def test_blocked(self): today = util.utcnow() two_weeks = today - timedelta(days=14) assert not station_blocked(CellShardGsm()) assert station_blocked(CellShardGsm(created=two_weeks, block_count=1)) assert station_blocked( CellShardGsm(created=today - timedelta(30), block_count=1)) assert not station_blocked( CellShardGsm(created=today - timedelta(45), block_count=1)) assert station_blocked( CellShardGsm(created=today - timedelta(45), block_count=2)) assert not station_blocked( CellShardGsm(created=today - timedelta(105), block_count=3)) assert station_blocked( CellShardGsm(created=two_weeks, block_last=today.date())) assert not station_blocked( CellShardGsm(created=two_weeks, block_last=two_weeks.date())) assert station_blocked( CellShardGsm(created=two_weeks, block_last=two_weeks.date()), two_weeks.date())
def test_blocked(self): today = util.utcnow() two_weeks = today - timedelta(days=14) assert not station_blocked(CellShardGsm()) assert station_blocked(CellShardGsm( created=two_weeks, block_count=1)) assert station_blocked(CellShardGsm( created=today - timedelta(30), block_count=1)) assert not station_blocked(CellShardGsm( created=today - timedelta(45), block_count=1)) assert station_blocked(CellShardGsm( created=today - timedelta(45), block_count=2)) assert not station_blocked(CellShardGsm( created=today - timedelta(105), block_count=3)) assert station_blocked(CellShardGsm( created=two_weeks, block_last=today.date())) assert not station_blocked(CellShardGsm( created=two_weeks, block_last=two_weeks.date())) assert station_blocked(CellShardGsm( created=two_weeks, block_last=two_weeks.date()), two_weeks.date())
def test_blocklist_skip(self, celery, redis, session): observations = self.obs_factory.build_batch(3) self.station_factory(created=self.now, block_first=self.ten_days.date(), block_last=self.today, block_count=1, **self.key(observations[0])) session.commit() self.queue_and_update(celery, observations) blocks = [] for obs in observations: cell = self.get_station(session, obs) if station_blocked(cell): blocks.append(cell) assert len(blocks) == 1 self.check_blocked(blocks[0], self.ten_days.date(), self.today, 1) self.check_areas(celery, [observations[1], observations[2]]) self.check_statcounter(redis, self.stat_obs_key, 3) self.check_statcounter(redis, self.stat_station_key, 2)
def query_cells(query, lookups, model, raven_client): # Given a location query and a list of lookup instances, query the # database and return a list of model objects. cellids = [lookup.cellid for lookup in lookups] if not cellids: # pragma: no cover return [] # load all fields used in score calculation and those we # need for the position load_fields = ('cellid', 'lat', 'lon', 'radius', 'region', 'samples', 'created', 'modified', 'last_seen', 'block_last', 'block_count') result = [] today = util.utcnow().date() try: shards = defaultdict(list) for lookup in lookups: shards[model.shard_model(lookup.radioType)].append(lookup.cellid) for shard, shard_cellids in shards.items(): columns = shard.__table__.c fields = [getattr(columns, f) for f in load_fields] rows = ( query.session.execute( select(fields) .where(columns.lat.isnot(None)) .where(columns.lon.isnot(None)) .where(columns.cellid.in_(shard_cellids))) ).fetchall() result.extend([row for row in rows if not station_blocked(row, today)]) except Exception: raven_client.captureException() return result
def test_blocklist_skip(self, celery, redis, session): observations = self.obs_factory.build_batch(3) self.station_factory( created=self.now, block_first=self.ten_days.date(), block_last=self.today, block_count=1, **self.key(observations[0]) ) session.commit() self.queue_and_update(celery, observations) blocks = [] for obs in observations: cell = self.get_station(session, obs) if station_blocked(cell): blocks.append(cell) assert len(blocks) == 1 self.check_blocked(blocks[0], self.ten_days.date(), self.today, 1) self.check_areas(celery, [observations[1], observations[2]]) self.check_statcounter(redis, self.stat_obs_key, 3) self.check_statcounter(redis, self.stat_station_key, 2)