def cluster_areas(areas, lookups, min_age=0):
"""
Cluster areas, treat each area as its own cluster.
"""
now = util.utcnow()
today = now.date()
# Create a dict of area ids mapped to their age and signal strength.
obs_data = {}
for lookup in lookups:
obs_data[decode_cellarea(lookup.areaid)] = (
max(abs(lookup.age or min_age), 1000),
lookup.signalStrength or MIN_CELL_SIGNAL[lookup.radioType])
clusters = []
for area in areas:
clusters.append(numpy.array([(
area.lat, area.lon, area.radius,
obs_data[area.areaid][0],
obs_data[area.areaid][1],
area_score(area, now),
encode_cellarea(*area.areaid),
bool(area.last_seen >= today))],
dtype=NETWORK_DTYPE))
return clusters
def cluster_areas(areas, lookups, min_age=0):
"""
Cluster areas, treat each area as its own cluster.
"""
now = util.utcnow()
today = now.date()
# Create a dict of area ids mapped to their age and signal strength.
obs_data = {}
for lookup in lookups:
obs_data[decode_cellarea(lookup.areaid)] = (
max(abs(lookup.age or min_age), 1000),
lookup.signalStrength or MIN_CELL_SIGNAL[lookup.radioType],
)
clusters = []
for area in areas:
clusters.append(
numpy.array(
[(
area.lat,
area.lon,
area.radius,
obs_data[area.areaid][0],
obs_data[area.areaid][1],
area_score(area, now),
encode_cellarea(*area.areaid, codec="base64"),
bool(area.last_seen is not None
and area.last_seen >= today),
)],
dtype=NETWORK_DTYPE,
))
return clusters
def update_areas(self, areaids):
"""Update the cell areas based on cell station records."""
areaid_set = set(areaids)
with self.task.db_session() as session:
# Fetch and lock existing cellarea rows
rows = session.execute(
select([self.area_table.c.areaid])
.where(self.area_table.c.areaid.in_(areaid_set))
.with_for_update()
).fetchall()
existing_areas = set(row[0] for row in rows)
# Update each cellarea row from cell tables
for areaid in areaid_set:
area_is_new = decode_cellarea(areaid) not in existing_areas
self.update_area(session, areaid, area_is_new)
def update_area(self, areaid):
# Select all cells in this area and derive a bounding box for them
radio, mcc, mnc, lac = decode_cellarea(areaid)
load_fields = ('lat', 'lon', 'radius', 'region',
'max_lat', 'max_lon', 'min_lat', 'min_lon')
shard = self.cell_model.shard_model(radio)
cells = (self.session.query(shard)
.options(load_only(*load_fields))
.filter(shard.radio == radio)
.filter(shard.mcc == mcc)
.filter(shard.mnc == mnc)
.filter(shard.lac == lac)
.filter(shard.lat.isnot(None))
.filter(shard.lon.isnot(None))).all()
area_query = (self.session.query(self.area_model)
.filter(self.area_model.areaid == areaid))
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
area_query.delete()
else:
# Otherwise update the area entry based on all the cells
area = area_query.first()
cell_extremes = numpy.array([
(numpy.nan if cell.max_lat is None else cell.max_lat,
numpy.nan if cell.max_lon is None else cell.max_lon)
for cell in cells] + [
(numpy.nan if cell.min_lat is None else cell.min_lat,
numpy.nan if cell.min_lon is None else cell.min_lon)
for cell in cells
], dtype=numpy.double)
max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0)
min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0)
ctr_lat, ctr_lon = centroid(
numpy.array([(c.lat, c.lon) for c in cells],
dtype=numpy.double))
radius = circle_radius(
ctr_lat, ctr_lon, max_lat, max_lon, min_lat, min_lon)
cell_radii = numpy.array([
(numpy.nan if cell.radius is None else cell.radius)
for cell in cells
], dtype=numpy.int32)
avg_cell_radius = int(round(numpy.nanmean(cell_radii)))
num_cells = len(cells)
region = self.region(ctr_lat, ctr_lon, mcc, cells)
if area is None:
stmt = self.area_model.__table__.insert(
mysql_on_duplicate='num_cells = num_cells' # no-op
).values(
areaid=areaid,
radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
radius=radius,
region=region,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
)
self.session.execute(stmt)
else:
area.modified = self.utcnow
area.lat = ctr_lat
area.lon = ctr_lon
area.radius = radius
area.region = region
area.avg_cell_radius = avg_cell_radius
area.num_cells = num_cells
def update_area(self, session, areaid, area_is_new):
# Select all cells in this area and derive a bounding box for them
radio, mcc, mnc, lac = decode_cellarea(areaid)
load_fields = (
"cellid",
"lat",
"lon",
"radius",
"region",
"last_seen",
"max_lat",
"max_lon",
"min_lat",
"min_lon",
)
shard = self.cell_model.shard_model(radio)
fields = [getattr(shard.__table__.c, f) for f in load_fields]
cells = session.execute(
select(fields)
.where(shard.__table__.c.radio == radio)
.where(shard.__table__.c.mcc == mcc)
.where(shard.__table__.c.mnc == mnc)
.where(shard.__table__.c.lac == lac)
.where(shard.__table__.c.lat.isnot(None))
.where(shard.__table__.c.lon.isnot(None))
).fetchall()
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
session.execute(
delete(self.area_table).where(self.area_table.c.areaid == areaid)
)
return
# Otherwise update the area entry based on all the cells
cell_extremes = numpy.array(
[
(
numpy.nan if cell.max_lat is None else cell.max_lat,
numpy.nan if cell.max_lon is None else cell.max_lon,
)
for cell in cells
]
+ [
(
numpy.nan if cell.min_lat is None else cell.min_lat,
numpy.nan if cell.min_lon is None else cell.min_lon,
)
for cell in cells
],
dtype=numpy.double,
)
max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0)
min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0)
ctr_lat, ctr_lon = numpy.array(
[(c.lat, c.lon) for c in cells], dtype=numpy.double
).mean(axis=0)
ctr_lat = float(ctr_lat)
ctr_lon = float(ctr_lon)
radius = circle_radius(ctr_lat, ctr_lon, max_lat, max_lon, min_lat, min_lon)
cell_radii = numpy.array(
[(numpy.nan if cell.radius is None else cell.radius) for cell in cells],
dtype=numpy.int32,
)
avg_cell_radius = int(round(numpy.nanmean(cell_radii)))
num_cells = len(cells)
region = self.region(ctr_lat, ctr_lon, mcc, cells)
last_seen = None
cell_last_seen = set(
[cell.last_seen for cell in cells if cell.last_seen is not None]
)
if cell_last_seen:
last_seen = max(cell_last_seen)
if area_is_new:
session.execute(
self.area_table.insert().values(
areaid=areaid,
radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
radius=radius,
region=region,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
last_seen=last_seen,
)
# If there was an unexpected insert, log warning instead of error
.prefix_with("IGNORE", dialect="mysql")
)
else:
session.execute(
self.area_table.update()
.where(self.area_table.c.areaid == areaid)
.values(
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
radius=radius,
region=region,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
last_seen=last_seen,
)
)
def update_area(self, session, areaid):
# Select all cells in this area and derive a bounding box for them
radio, mcc, mnc, lac = decode_cellarea(areaid)
load_fields = ('cellid', 'lat', 'lon', 'radius', 'region', 'last_seen',
'max_lat', 'max_lon', 'min_lat', 'min_lon')
shard = self.cell_model.shard_model(radio)
fields = [getattr(shard.__table__.c, f) for f in load_fields]
cells = session.execute(
select(fields)
.where(shard.__table__.c.radio == radio)
.where(shard.__table__.c.mcc == mcc)
.where(shard.__table__.c.mnc == mnc)
.where(shard.__table__.c.lac == lac)
.where(shard.__table__.c.lat.isnot(None))
.where(shard.__table__.c.lon.isnot(None))
).fetchall()
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
session.execute(
delete(self.area_table)
.where(self.area_table.c.areaid == areaid)
)
return
# Otherwise update the area entry based on all the cells
area = session.execute(
select([self.area_table.c.areaid,
self.area_table.c.modified,
self.area_table.c.lat,
self.area_table.c.lon,
self.area_table.c.radius,
self.area_table.c.region,
self.area_table.c.avg_cell_radius,
self.area_table.c.num_cells,
self.area_table.c.last_seen,
])
.where(self.area_table.c.areaid == areaid)
).fetchone()
cell_extremes = numpy.array([
(numpy.nan if cell.max_lat is None else cell.max_lat,
numpy.nan if cell.max_lon is None else cell.max_lon)
for cell in cells] + [
(numpy.nan if cell.min_lat is None else cell.min_lat,
numpy.nan if cell.min_lon is None else cell.min_lon)
for cell in cells
], dtype=numpy.double)
max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0)
min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0)
ctr_lat, ctr_lon = numpy.array(
[(c.lat, c.lon) for c in cells],
dtype=numpy.double).mean(axis=0)
ctr_lat = float(ctr_lat)
ctr_lon = float(ctr_lon)
radius = circle_radius(
ctr_lat, ctr_lon, max_lat, max_lon, min_lat, min_lon)
cell_radii = numpy.array([
(numpy.nan if cell.radius is None else cell.radius)
for cell in cells
], dtype=numpy.int32)
avg_cell_radius = int(round(numpy.nanmean(cell_radii)))
num_cells = len(cells)
region = self.region(ctr_lat, ctr_lon, mcc, cells)
last_seen = None
cell_last_seen = set([cell.last_seen for cell in cells
if cell.last_seen is not None])
if cell_last_seen:
last_seen = max(cell_last_seen)
if area is None:
session.execute(
self.area_table.insert(
mysql_on_duplicate='num_cells = num_cells' # no-op
).values(
areaid=areaid,
radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
radius=radius,
region=region,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
last_seen=last_seen,
)
)
else:
session.execute(
self.area_table.update()
.where(self.area_table.c.areaid == areaid)
.values(
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
radius=radius,
region=region,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
last_seen=last_seen,
)
)