def query_cells(query, lookups, model, raven_client):
# Given a location query and a list of lookup instances, query the
# database and return a list of model objects.
cellids = [lookup.cellid for lookup in lookups]
if not cellids: # pragma: no cover
return []
# load all fields used in score calculation and those we
# need for the position
load_fields = ('cellid', 'lat', 'lon', 'radius', 'region', 'samples',
'created', 'modified', 'last_seen', 'block_last',
'block_count')
result = []
today = util.utcnow().date()
try:
shards = defaultdict(list)
for lookup in lookups:
shards[model.shard_model(lookup.radioType)].append(lookup.cellid)
for shard, shard_cellids in shards.items():
columns = shard.__table__.c
fields = [getattr(columns, f) for f in load_fields]
rows = (query.session.execute(
select(fields).where(columns.lat.isnot(None)).where(
columns.lon.isnot(None)).where(
columns.cellid.in_(shard_cellids)))).fetchall()
result.extend(
[row for row in rows if not station_blocked(row, today)])
except Exception:
raven_client.captureException()
return result
def query_macs(query, lookups, raven_client, db_model):
macs = [lookup.mac for lookup in lookups]
if not macs: # pragma: no cover
return []
# load all fields used in score calculation and those we
# need for the position or region
load_fields = ('mac', 'lat', 'lon', 'radius', 'region', 'samples',
'created', 'modified', 'last_seen', 'block_last',
'block_count')
result = []
today = util.utcnow().date()
try:
shards = defaultdict(list)
for mac in macs:
shards[db_model.shard_model(mac)].append(mac)
for shard, shard_macs in shards.items():
columns = shard.__table__.c
fields = [getattr(columns, f) for f in load_fields]
rows = (query.session.execute(
select(fields).where(columns.lat.isnot(None)).where(
columns.lon.isnot(None)).where(
columns.mac.in_(shard_macs)))).fetchall()
result.extend(
[row for row in rows if not station_blocked(row, today)])
except Exception:
raven_client.captureException()
return result
def query_macs(query, lookups, raven_client, db_model):
macs = [lookup.mac for lookup in lookups]
if not macs: # pragma: no cover
return []
# load all fields used in score calculation and those we
# need for the position or region
load_fields = ('mac', 'lat', 'lon', 'radius', 'region', 'samples',
'created', 'modified', 'last_seen',
'block_last', 'block_count')
result = []
today = util.utcnow().date()
try:
shards = defaultdict(list)
for mac in macs:
shards[db_model.shard_model(mac)].append(mac)
for shard, shard_macs in shards.items():
columns = shard.__table__.c
fields = [getattr(columns, f) for f in load_fields]
rows = (
query.session.execute(
select(fields)
.where(columns.lat.isnot(None))
.where(columns.lon.isnot(None))
.where(columns.mac.in_(shard_macs)))
).fetchall()
result.extend([row for row in rows
if not station_blocked(row, today)])
except Exception:
raven_client.captureException()
return result
def query_stations(self, session, shard, shard_values):
blocklist = {}
stations = {}
keys = list(shard_values.keys())
rows = self.query_shard(session, shard, keys)
for row in rows:
unique_key = row.unique_key
stations[unique_key] = row
blocklist[unique_key] = station_blocked(row, self.today)
return (blocklist, stations)
def query_stations(self, session, shard, shard_values):
blocklist = {}
stations = {}
keys = list(shard_values.keys())
rows = self.query_shard(session, shard, keys)
for row in rows:
unique_key = row.unique_key
stations[unique_key] = row
blocklist[unique_key] = station_blocked(row, self.today)
return (blocklist, stations)
def test_blocked(self):
today = util.utcnow()
two_weeks = today - timedelta(days=14)
assert not station_blocked(CellShardGsm())
assert station_blocked(CellShardGsm(created=two_weeks, block_count=1))
assert station_blocked(
CellShardGsm(created=today - timedelta(30), block_count=1))
assert not station_blocked(
CellShardGsm(created=today - timedelta(45), block_count=1))
assert station_blocked(
CellShardGsm(created=today - timedelta(45), block_count=2))
assert not station_blocked(
CellShardGsm(created=today - timedelta(105), block_count=3))
assert station_blocked(
CellShardGsm(created=two_weeks, block_last=today.date()))
assert not station_blocked(
CellShardGsm(created=two_weeks, block_last=two_weeks.date()))
assert station_blocked(
CellShardGsm(created=two_weeks, block_last=two_weeks.date()),
two_weeks.date())
def test_blocked(self):
today = util.utcnow()
two_weeks = today - timedelta(days=14)
assert not station_blocked(CellShardGsm())
assert station_blocked(CellShardGsm(
created=two_weeks, block_count=1))
assert station_blocked(CellShardGsm(
created=today - timedelta(30), block_count=1))
assert not station_blocked(CellShardGsm(
created=today - timedelta(45), block_count=1))
assert station_blocked(CellShardGsm(
created=today - timedelta(45), block_count=2))
assert not station_blocked(CellShardGsm(
created=today - timedelta(105), block_count=3))
assert station_blocked(CellShardGsm(
created=two_weeks, block_last=today.date()))
assert not station_blocked(CellShardGsm(
created=two_weeks, block_last=two_weeks.date()))
assert station_blocked(CellShardGsm(
created=two_weeks, block_last=two_weeks.date()), two_weeks.date())
def test_blocklist_skip(self, celery, redis, session):
observations = self.obs_factory.build_batch(3)
self.station_factory(created=self.now,
block_first=self.ten_days.date(),
block_last=self.today,
block_count=1,
**self.key(observations[0]))
session.commit()
self.queue_and_update(celery, observations)
blocks = []
for obs in observations:
cell = self.get_station(session, obs)
if station_blocked(cell):
blocks.append(cell)
assert len(blocks) == 1
self.check_blocked(blocks[0], self.ten_days.date(), self.today, 1)
self.check_areas(celery, [observations[1], observations[2]])
self.check_statcounter(redis, self.stat_obs_key, 3)
self.check_statcounter(redis, self.stat_station_key, 2)
def query_cells(query, lookups, model, raven_client):
# Given a location query and a list of lookup instances, query the
# database and return a list of model objects.
cellids = [lookup.cellid for lookup in lookups]
if not cellids: # pragma: no cover
return []
# load all fields used in score calculation and those we
# need for the position
load_fields = ('cellid', 'lat', 'lon', 'radius', 'region', 'samples',
'created', 'modified', 'last_seen',
'block_last', 'block_count')
result = []
today = util.utcnow().date()
try:
shards = defaultdict(list)
for lookup in lookups:
shards[model.shard_model(lookup.radioType)].append(lookup.cellid)
for shard, shard_cellids in shards.items():
columns = shard.__table__.c
fields = [getattr(columns, f) for f in load_fields]
rows = (
query.session.execute(
select(fields)
.where(columns.lat.isnot(None))
.where(columns.lon.isnot(None))
.where(columns.cellid.in_(shard_cellids)))
).fetchall()
result.extend([row for row in rows
if not station_blocked(row, today)])
except Exception:
raven_client.captureException()
return result
def test_blocklist_skip(self, celery, redis, session):
observations = self.obs_factory.build_batch(3)
self.station_factory(
created=self.now,
block_first=self.ten_days.date(),
block_last=self.today,
block_count=1,
**self.key(observations[0])
)
session.commit()
self.queue_and_update(celery, observations)
blocks = []
for obs in observations:
cell = self.get_station(session, obs)
if station_blocked(cell):
blocks.append(cell)
assert len(blocks) == 1
self.check_blocked(blocks[0], self.ten_days.date(), self.today, 1)
self.check_areas(celery, [observations[1], observations[2]])
self.check_statcounter(redis, self.stat_obs_key, 3)
self.check_statcounter(redis, self.stat_station_key, 2)
