def aggregate_obs(self):
positions = numpy.array(
[(obs.lat, obs.lon) for obs in self.observations],
dtype=numpy.double)
max_lat, max_lon = positions.max(axis=0)
min_lat, min_lon = positions.min(axis=0)
box_distance = distance(min_lat, min_lon, max_lat, max_lon)
if box_distance > self.MAX_DIST_METERS:
return None
weights = numpy.array(
[obs.weight for obs in self.observations],
dtype=numpy.double)
lat, lon = numpy.average(positions, axis=0, weights=weights)
lat = float(lat)
lon = float(lon)
radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon)
region = GEOCODER.region(lat, lon)
samples, weight = self.bounded_samples_weight(
len(self.observations), float(weights.sum()))
return {
'positions': positions, 'weights': weights,
'lat': lat, 'lon': lon,
'max_lat': float(max_lat), 'min_lat': float(min_lat),
'max_lon': float(max_lon), 'min_lon': float(min_lon),
'radius': radius, 'region': region,
'samples': samples, 'weight': weight,
}
def _update_area(bind, shard_id, areaid, circle_radius):
radio, mcc, mnc, lac = CELLAREA_STRUCT.unpack(codecs.decode(areaid, 'hex'))
row = bind.execute(
sa.text(
stmt_select_area.format(id=shard_id,
radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac))).fetchone()
num_cells = int(row.num_cells)
if num_cells == 0:
op.execute(
sa.text(
stmt_delete_area.format(radio=radio, mcc=mcc, mnc=mnc,
lac=lac)))
else:
radius = circle_radius(float(row.lat), float(row.lon),
float(row.max_lat), float(row.max_lon),
float(row.min_lat), float(row.min_lon))
avg_cell_radius = int(round(row.avg_cell_radius))
op.execute(
sa.text(
stmt_update_area.format(
radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
lat=float(row.lat),
lon=float(row.lon),
radius=radius,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
)))
def aggregate_station_obs(self):
station = self.station
obs_data = self.obs_data
def get_nan(name):
value = getattr(station, name, None)
return numpy.nan if value is None else value
positions = numpy.append(obs_data['positions'], [
(get_nan('lat'), get_nan('lon')),
(get_nan('max_lat'), get_nan('max_lon')),
(get_nan('min_lat'), get_nan('min_lon')),
],
axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
if station.lat is None or station.lon is None:
old_weight = 0.0
else:
old_weight = min((station.weight or 0.0), self.MAX_OLD_WEIGHT)
lat = ((obs_data['lat'] * obs_data['weight'] +
(station.lat or 0.0) * old_weight) /
(obs_data['weight'] + old_weight))
lon = ((obs_data['lon'] * obs_data['weight'] +
(station.lon or 0.0) * old_weight) /
(obs_data['weight'] + old_weight))
radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon)
region = station.region
if (region and not GEOCODER.in_region(lat, lon, region)):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(lat, lon)
samples, weight = self.bounded_samples_weight(
(station.samples or 0) + obs_data['samples'],
(station.weight or 0.0) + obs_data['weight'])
return {
'lat': lat,
'lon': lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'radius': radius,
'region': region,
'samples': samples,
'weight': weight,
}
def aggregate_station_obs(self):
station = self.station
obs_data = self.obs_data
def get_nan(name):
value = getattr(station, name, None)
return numpy.nan if value is None else value
positions = numpy.append(obs_data['positions'], [
(get_nan('lat'), get_nan('lon')),
(get_nan('max_lat'), get_nan('max_lon')),
(get_nan('min_lat'), get_nan('min_lon')),
], axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
if station.lat is None or station.lon is None:
old_weight = 0.0
else:
old_weight = min((station.weight or 0.0), self.MAX_OLD_WEIGHT)
lat = ((obs_data['lat'] * obs_data['weight'] +
(station.lat or 0.0) * old_weight) /
(obs_data['weight'] + old_weight))
lon = ((obs_data['lon'] * obs_data['weight'] +
(station.lon or 0.0) * old_weight) /
(obs_data['weight'] + old_weight))
radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon)
region = station.region
if (region and not GEOCODER.in_region(lat, lon, region)):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(lat, lon)
samples, weight = self.bounded_samples_weight(
(station.samples or 0) + obs_data['samples'],
(station.weight or 0.0) + obs_data['weight'])
return {
'lat': lat, 'lon': lon,
'max_lat': float(max_lat), 'min_lat': float(min_lat),
'max_lon': float(max_lon), 'min_lon': float(min_lon),
'radius': radius, 'region': region,
'samples': samples, 'weight': weight,
}
def _update_area(bind, shard_id, areaid, circle_radius):
radio, mcc, mnc, lac = CELLAREA_STRUCT.unpack(codecs.decode(areaid, 'hex'))
row = bind.execute(sa.text(stmt_select_area.format(
id=shard_id, radio=radio, mcc=mcc, mnc=mnc, lac=lac))).fetchone()
num_cells = int(row.num_cells)
if num_cells == 0:
op.execute(sa.text(stmt_delete_area.format(
radio=radio, mcc=mcc, mnc=mnc, lac=lac)))
else:
radius = circle_radius(
float(row.lat), float(row.lon),
float(row.max_lat), float(row.max_lon),
float(row.min_lat), float(row.min_lon))
avg_cell_radius = int(round(row.avg_cell_radius))
op.execute(sa.text(stmt_update_area.format(
radio=radio, mcc=mcc, mnc=mnc, lac=lac,
lat=float(row.lat), lon=float(row.lon), radius=radius,
avg_cell_radius=avg_cell_radius, num_cells=num_cells,
)))
def aggregate_obs(self):
positions = numpy.array([(obs.lat, obs.lon)
for obs in self.observations],
dtype=numpy.double)
max_lat, max_lon = positions.max(axis=0)
min_lat, min_lon = positions.min(axis=0)
box_distance = distance(min_lat, min_lon, max_lat, max_lon)
if box_distance > self.MAX_DIST_METERS:
return None
weights = numpy.array([obs.weight for obs in self.observations],
dtype=numpy.double)
lat, lon = numpy.average(positions, axis=0, weights=weights)
lat = float(lat)
lon = float(lon)
radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon)
region = GEOCODER.region(lat, lon)
samples, weight = self.bounded_samples_weight(len(self.observations),
float(weights.sum()))
return {
'positions': positions,
'weights': weights,
'lat': lat,
'lon': lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'radius': radius,
'region': region,
'samples': samples,
'weight': weight,
}
def station_values(self, station_key, shard_station, observations):
"""
Return two-tuple of status, value dict where status is one of:
`new`, `new_moving`, `moving`, `changed`.
"""
# cases:
# we always get a station key and observations
# 0. observations disagree
# 0.a. no shard station, return new_moving
# 0.b. shard station, return moving
# 1. no shard station
# 1.a. obs agree -> return new
# 2. shard station
# 2.a. obs disagree -> return moving
# 2.b. obs agree -> return changed
created = self.utcnow
values = self._base_station_values(station_key, observations)
obs_length = len(observations)
obs_positions = numpy.array([(obs.lat, obs.lon)
for obs in observations],
dtype=numpy.double)
obs_new_lat, obs_new_lon = centroid(obs_positions)
obs_max_lat, obs_max_lon = numpy.nanmax(obs_positions, axis=0)
obs_min_lat, obs_min_lon = numpy.nanmin(obs_positions, axis=0)
obs_box_dist = distance(obs_min_lat, obs_min_lon, obs_max_lat,
obs_max_lon)
if obs_box_dist > self.max_dist_meters:
# the new observations are already too far apart
if not shard_station:
values.update({
'created': created,
'block_first': self.today,
'block_last': self.today,
'block_count': 1,
})
return ('new_moving', values)
else:
block_count = shard_station.block_count or 0
values.update({
'lat':
None,
'lon':
None,
'max_lat':
None,
'min_lat':
None,
'max_lon':
None,
'min_lon':
None,
'radius':
None,
'region':
shard_station.region,
'samples':
None,
'source':
None,
'block_first':
shard_station.block_first or self.today,
'block_last':
self.today,
'block_count':
block_count + 1,
})
return ('moving', values)
if shard_station is None:
# totally new station, only agreeing observations
radius = circle_radius(obs_new_lat, obs_new_lon, obs_max_lat,
obs_max_lon, obs_min_lat, obs_min_lon)
values.update({
'created': created,
'lat': obs_new_lat,
'lon': obs_new_lon,
'max_lat': float(obs_max_lat),
'min_lat': float(obs_min_lat),
'max_lon': float(obs_max_lon),
'min_lon': float(obs_min_lon),
'radius': radius,
'region': GEOCODER.region(obs_new_lat, obs_new_lon),
'samples': obs_length,
'source': None,
})
return ('new', values)
else:
# shard_station + new observations
positions = numpy.append(obs_positions, [
(numpy.nan if shard_station.lat is None else shard_station.lat,
numpy.nan
if shard_station.lon is None else shard_station.lon),
(numpy.nan if shard_station.max_lat is None else
shard_station.max_lat, numpy.nan
if shard_station.max_lon is None else shard_station.max_lon),
(numpy.nan if shard_station.min_lat is None else
shard_station.min_lat, numpy.nan
if shard_station.min_lon is None else shard_station.min_lon),
],
axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
box_dist = distance(min_lat, min_lon, max_lat, max_lon)
if box_dist > self.max_dist_meters:
# shard_station + disagreeing observations
block_count = shard_station.block_count or 0
values.update({
'lat':
None,
'lon':
None,
'max_lat':
None,
'min_lat':
None,
'max_lon':
None,
'min_lon':
None,
'radius':
None,
'region':
shard_station.region,
'samples':
None,
'source':
None,
'block_first':
shard_station.block_first or self.today,
'block_last':
self.today,
'block_count':
block_count + 1,
})
return ('moving', values)
else:
# shard_station + agreeing observations
if shard_station.lat is None or shard_station.lon is None:
old_weight = 0
else:
old_weight = min((shard_station.samples or 0),
self.MAX_OLD_OBSERVATIONS)
new_lat = ((obs_new_lat * obs_length +
(shard_station.lat or 0.0) * old_weight) /
(obs_length + old_weight))
new_lon = ((obs_new_lon * obs_length +
(shard_station.lon or 0.0) * old_weight) /
(obs_length + old_weight))
samples = (shard_station.samples or 0) + obs_length
radius = circle_radius(new_lat, new_lon, max_lat, max_lon,
min_lat, min_lon)
region = shard_station.region
if (region
and not GEOCODER.in_region(new_lat, new_lon, region)):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(new_lat, new_lon)
values.update({
'lat': new_lat,
'lon': new_lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'radius': radius,
'region': region,
'samples': samples,
'source': None,
# use the exact same keys as in the moving case
'block_first': shard_station.block_first,
'block_last': shard_station.block_last,
'block_count': shard_station.block_count,
})
return ('changed', values)
return (None, None) # pragma: no cover
def update_area(self, areaid):
# Select all cells in this area and derive a bounding box for them
radio, mcc, mnc, lac = decode_cellarea(areaid)
load_fields = ('lat', 'lon', 'radius', 'region',
'max_lat', 'max_lon', 'min_lat', 'min_lon')
shard = self.cell_model.shard_model(radio)
cells = (self.session.query(shard)
.options(load_only(*load_fields))
.filter(shard.radio == radio)
.filter(shard.mcc == mcc)
.filter(shard.mnc == mnc)
.filter(shard.lac == lac)
.filter(shard.lat.isnot(None))
.filter(shard.lon.isnot(None))).all()
area_query = (self.session.query(self.area_model)
.filter(self.area_model.areaid == areaid))
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
area_query.delete()
else:
# Otherwise update the area entry based on all the cells
area = area_query.first()
cell_extremes = numpy.array([
(numpy.nan if cell.max_lat is None else cell.max_lat,
numpy.nan if cell.max_lon is None else cell.max_lon)
for cell in cells] + [
(numpy.nan if cell.min_lat is None else cell.min_lat,
numpy.nan if cell.min_lon is None else cell.min_lon)
for cell in cells
], dtype=numpy.double)
max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0)
min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0)
ctr_lat, ctr_lon = centroid(
numpy.array([(c.lat, c.lon) for c in cells],
dtype=numpy.double))
radius = circle_radius(
ctr_lat, ctr_lon, max_lat, max_lon, min_lat, min_lon)
cell_radii = numpy.array([
(numpy.nan if cell.radius is None else cell.radius)
for cell in cells
], dtype=numpy.int32)
avg_cell_radius = int(round(numpy.nanmean(cell_radii)))
num_cells = len(cells)
region = self.region(ctr_lat, ctr_lon, mcc, cells)
if area is None:
stmt = self.area_model.__table__.insert(
mysql_on_duplicate='num_cells = num_cells' # no-op
).values(
areaid=areaid,
radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
radius=radius,
region=region,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
)
self.session.execute(stmt)
else:
area.modified = self.utcnow
area.lat = ctr_lat
area.lon = ctr_lon
area.radius = radius
area.region = region
area.avg_cell_radius = avg_cell_radius
area.num_cells = num_cells
def station_values(self, station_key, shard_station, observations):
"""
Return two-tuple of status, value dict where status is one of:
`new`, `new_moving`, `moving`, `changed`.
"""
# cases:
# we always get a station key and observations
# 0. observations disagree
# 0.a. no shard station, return new_moving
# 0.b. shard station, return moving
# 1. no shard station
# 1.a. obs agree -> return new
# 2. shard station
# 2.a. obs disagree -> return moving
# 2.b. obs agree -> return changed
created = self.utcnow
values = {
'mac': station_key,
'modified': self.utcnow,
}
obs_length = len(observations)
obs_positions = numpy.array(
[(obs.lat, obs.lon) for obs in observations],
dtype=numpy.double)
obs_new_lat, obs_new_lon = centroid(obs_positions)
obs_max_lat, obs_max_lon = numpy.nanmax(obs_positions, axis=0)
obs_min_lat, obs_min_lon = numpy.nanmin(obs_positions, axis=0)
obs_box_dist = distance(obs_min_lat, obs_min_lon,
obs_max_lat, obs_max_lon)
if obs_box_dist > self.max_dist_meters:
# the new observations are already too far apart
if not shard_station:
values.update({
'created': created,
'block_first': self.today,
'block_last': self.today,
'block_count': 1,
})
return ('new_moving', values)
else:
block_count = shard_station.block_count or 0
values.update({
'lat': None,
'lon': None,
'max_lat': None,
'min_lat': None,
'max_lon': None,
'min_lon': None,
'country': shard_station.country,
'radius': None,
'samples': None,
'source': None,
'block_last': self.today,
'block_count': block_count + 1,
})
return ('moving', values)
if shard_station is None:
# totally new station, only agreeing observations
radius = circle_radius(
obs_new_lat, obs_new_lon,
obs_max_lat, obs_max_lon, obs_min_lat, obs_min_lon)
values.update({
'created': created,
'lat': obs_new_lat,
'lon': obs_new_lon,
'max_lat': float(obs_max_lat),
'min_lat': float(obs_min_lat),
'max_lon': float(obs_max_lon),
'min_lon': float(obs_min_lon),
'country': country_for_location(obs_new_lat, obs_new_lon),
'radius': radius,
'samples': obs_length,
'source': None,
})
return ('new', values)
else:
# shard_station + new observations
positions = numpy.append(obs_positions, [
(numpy.nan if shard_station.lat is None
else shard_station.lat,
numpy.nan if shard_station.lon is None
else shard_station.lon),
(numpy.nan if shard_station.max_lat is None
else shard_station.max_lat,
numpy.nan if shard_station.max_lon is None
else shard_station.max_lon),
(numpy.nan if shard_station.min_lat is None
else shard_station.min_lat,
numpy.nan if shard_station.min_lon is None
else shard_station.min_lon),
], axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
box_dist = distance(min_lat, min_lon, max_lat, max_lon)
if box_dist > self.max_dist_meters:
# shard_station + disagreeing observations
block_count = shard_station.block_count or 0
values.update({
'lat': None,
'lon': None,
'max_lat': None,
'min_lat': None,
'max_lon': None,
'min_lon': None,
'country': shard_station.country,
'radius': None,
'samples': None,
'source': None,
'block_last': self.today,
'block_count': block_count + 1,
})
return ('moving', values)
else:
# shard_station + agreeing observations
if shard_station.lat is None or shard_station.lon is None:
old_weight = 0
else:
old_weight = min((shard_station.samples or 0),
self.MAX_OLD_OBSERVATIONS)
new_lat = ((obs_new_lat * obs_length +
(shard_station.lat or 0.0) * old_weight) /
(obs_length + old_weight))
new_lon = ((obs_new_lon * obs_length +
(shard_station.lon or 0.0) * old_weight) /
(obs_length + old_weight))
samples = (shard_station.samples or 0) + obs_length
radius = circle_radius(
new_lat, new_lon, max_lat, max_lon, min_lat, min_lon)
country = shard_station.country
if (country and not country_matches_location(
new_lat, new_lon, country)):
# reset country if it no longer matches
country = None
if not country:
country = country_for_location(new_lat, new_lon)
values.update({
'lat': new_lat,
'lon': new_lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'country': country,
'radius': radius,
'samples': samples,
'source': None,
# use the exact same keys as in the moving case
'block_last': shard_station.block_last,
'block_count': shard_station.block_count,
})
return ('changed', values)
return (None, None) # pragma: no cover
def new_station_values(self, station, station_key,
first_blocked, observations):
# This function returns a 3-tuple, the first element is True,
# if the station was found to be moving.
# The second element is either None or a dict of values,
# if the station is new and should result in a table insert
# The third element is either None or a dict of values
# if the station did exist and should be updated
obs_length = len(observations)
obs_positions = numpy.array(
[(obs.lat, obs.lon) for obs in observations],
dtype=numpy.double)
obs_lat, obs_lon = centroid(obs_positions)
values = {
'modified': self.utcnow,
}
values.update(station_key.__dict__)
if self.station_type == 'cell':
# pass on extra psc column which is not actually part
# of the stations hash key
values['psc'] = observations[-1].psc
created = self.utcnow
if station is None:
if first_blocked:
# if the station did previously exist, retain at least the
# time it was first put on a blocklist as the creation date
created = first_blocked
values.update({
'created': created,
'range': 0,
'total_measures': 0,
})
if (station is not None and
station.lat is not None and station.lon is not None):
obs_positions = numpy.append(obs_positions, [
(station.lat, station.lon),
(numpy.nan if station.max_lat is None else station.max_lat,
numpy.nan if station.max_lon is None else station.max_lon),
(numpy.nan if station.min_lat is None else station.min_lat,
numpy.nan if station.min_lon is None else station.min_lon),
], axis=0)
existing_station = True
else:
values['lat'] = obs_lat
values['lon'] = obs_lon
existing_station = False
max_lat, max_lon = numpy.nanmax(obs_positions, axis=0)
min_lat, min_lon = numpy.nanmin(obs_positions, axis=0)
# calculate sphere-distance from opposite corners of
# bounding box containing current location estimate
# and new observations; if too big, station is moving
box_dist = distance(min_lat, min_lon, max_lat, max_lon)
# TODO: If we get a too large box_dist, we should not create
# a new station record with the impossibly big distance,
# so moving the box_dist > self.max_dist_meters here
if existing_station:
if box_dist > self.max_dist_meters:
# Signal a moving station and return early without updating
# the station since it will be deleted by caller momentarily
return (True, None, None)
# limit the maximum weight of the old station estimate
old_weight = min(station.total_measures,
self.MAX_OLD_OBSERVATIONS)
new_weight = old_weight + obs_length
values['lat'] = ((station.lat * old_weight) +
(obs_lat * obs_length)) / new_weight
values['lon'] = ((station.lon * old_weight) +
(obs_lon * obs_length)) / new_weight
# increase total counter
if station is not None:
values['total_measures'] = station.total_measures + obs_length
else:
values['total_measures'] = obs_length
# update max/min lat/lon columns
values['min_lat'] = float(min_lat)
values['min_lon'] = float(min_lon)
values['max_lat'] = float(max_lat)
values['max_lon'] = float(max_lon)
# give radio-range estimate between extreme values and centroid
values['range'] = circle_radius(
values['lat'], values['lon'],
max_lat, max_lon, min_lat, min_lon)
if station is None:
# return new values
return (False, values, None)
else:
# return updated values, remove station from session
self.session.expunge(station)
return (False, None, values)
def update_area(self, area_key):
# Select all cells in this area and derive a bounding box for them
cell_query = (self.cell_model.querykey(self.session, area_key)
.filter(self.cell_model.lat.isnot(None))
.filter(self.cell_model.lon.isnot(None)))
cells = cell_query.all()
area_query = self.area_model.querykey(self.session, area_key)
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
area_query.delete()
else:
# Otherwise update the area entry based on all the cells
area = area_query.first()
cell_extremes = numpy.array([
(numpy.nan if cell.max_lat is None else cell.max_lat,
numpy.nan if cell.max_lon is None else cell.max_lon)
for cell in cells] + [
(numpy.nan if cell.min_lat is None else cell.min_lat,
numpy.nan if cell.min_lon is None else cell.min_lon)
for cell in cells
], dtype=numpy.double)
max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0)
min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0)
ctr_lat, ctr_lon = centroid(
numpy.array([(c.lat, c.lon) for c in cells],
dtype=numpy.double))
radius = circle_radius(
ctr_lat, ctr_lon,
max_lat, max_lon, min_lat, min_lon)
# Now create or update the area
cell_ranges = numpy.array([
(numpy.nan if cell.range is None else cell.range)
for cell in cells
], dtype=numpy.int32)
avg_cell_range = int(round(numpy.nanmean(cell_ranges)))
num_cells = len(cells)
if area is None:
stmt = self.area_model.__table__.insert(
mysql_on_duplicate='num_cells = num_cells' # no-op
).values(
created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
range=radius,
avg_cell_range=avg_cell_range,
num_cells=num_cells,
**area_key.__dict__
)
self.session.execute(stmt)
else:
area.modified = self.utcnow
area.lat = ctr_lat
area.lon = ctr_lon
area.range = radius
area.avg_cell_range = avg_cell_range
area.num_cells = num_cells
def station_values(self, station_key, shard_station, observations):
"""
Return two-tuple of status, value dict where status is one of:
`new`, `new_moving`, `moving`, `changed`.
"""
# cases:
# we always get a station key and observations
# 0. observations disagree
# 0.a. no shard station, return new_moving
# 0.b. shard station, return moving
# 1. no shard station
# 1.a. obs agree -> return new
# 2. shard station
# 2.a. obs disagree -> return moving
# 2.b. obs agree -> return changed
created = self.utcnow
values = self._base_station_values(station_key, observations)
obs_positions = numpy.array(
[(obs.lat, obs.lon) for obs in observations],
dtype=numpy.double)
obs_length = len(observations)
obs_weights = numpy.array(
[obs.weight for obs in observations],
dtype=numpy.double)
obs_weight = float(obs_weights.sum())
obs_new_lat, obs_new_lon = numpy.average(
obs_positions, axis=0, weights=obs_weights)
obs_new_lat = float(obs_new_lat)
obs_new_lon = float(obs_new_lon)
obs_max_lat, obs_max_lon = obs_positions.max(axis=0)
obs_min_lat, obs_min_lon = obs_positions.min(axis=0)
obs_box_dist = distance(obs_min_lat, obs_min_lon,
obs_max_lat, obs_max_lon)
if obs_box_dist > self.max_dist_meters:
# the new observations are already too far apart
if not shard_station:
values.update({
'created': created,
'block_first': self.today,
'block_last': self.today,
'block_count': 1,
})
return ('new_moving', values)
else:
block_count = shard_station.block_count or 0
values.update({
'lat': None,
'lon': None,
'max_lat': None,
'min_lat': None,
'max_lon': None,
'min_lon': None,
'radius': None,
'region': shard_station.region,
'samples': None,
'source': None,
'weight': None,
'block_first': shard_station.block_first or self.today,
'block_last': self.today,
'block_count': block_count + 1,
})
return ('moving', values)
if shard_station is None:
# totally new station, only agreeing observations
radius = circle_radius(
obs_new_lat, obs_new_lon,
obs_max_lat, obs_max_lon, obs_min_lat, obs_min_lon)
values.update({
'created': created,
'lat': obs_new_lat,
'lon': obs_new_lon,
'max_lat': float(obs_max_lat),
'min_lat': float(obs_min_lat),
'max_lon': float(obs_max_lon),
'min_lon': float(obs_min_lon),
'radius': radius,
'region': GEOCODER.region(obs_new_lat, obs_new_lon),
'samples': obs_length,
'source': None,
'weight': obs_weight,
})
return ('new', values)
else:
# shard_station + new observations
positions = numpy.append(obs_positions, [
(numpy.nan if shard_station.lat is None
else shard_station.lat,
numpy.nan if shard_station.lon is None
else shard_station.lon),
(numpy.nan if shard_station.max_lat is None
else shard_station.max_lat,
numpy.nan if shard_station.max_lon is None
else shard_station.max_lon),
(numpy.nan if shard_station.min_lat is None
else shard_station.min_lat,
numpy.nan if shard_station.min_lon is None
else shard_station.min_lon),
], axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
box_dist = distance(min_lat, min_lon, max_lat, max_lon)
if box_dist > self.max_dist_meters:
# shard_station + disagreeing observations
block_count = shard_station.block_count or 0
values.update({
'lat': None,
'lon': None,
'max_lat': None,
'min_lat': None,
'max_lon': None,
'min_lon': None,
'radius': None,
'region': shard_station.region,
'samples': None,
'source': None,
'weight': None,
'block_first': shard_station.block_first or self.today,
'block_last': self.today,
'block_count': block_count + 1,
})
return ('moving', values)
else:
# shard_station + agreeing observations
if shard_station.lat is None or shard_station.lon is None:
old_weight = 0
else:
old_weight = min((shard_station.weight or 0.0),
self.MAX_OLD_WEIGHT)
new_lat = ((obs_new_lat * obs_weight +
(shard_station.lat or 0.0) * old_weight) /
(obs_weight + old_weight))
new_lon = ((obs_new_lon * obs_weight +
(shard_station.lon or 0.0) * old_weight) /
(obs_weight + old_weight))
# put in maximum value to avoid overflow of DB column
samples = min((shard_station.samples or 0) + obs_length,
4294967295)
weight = min((shard_station.weight or 0.0) + obs_weight,
1000000000.0)
radius = circle_radius(
new_lat, new_lon, max_lat, max_lon, min_lat, min_lon)
region = shard_station.region
if (region and not GEOCODER.in_region(
new_lat, new_lon, region)):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(new_lat, new_lon)
values.update({
'lat': new_lat,
'lon': new_lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'radius': radius,
'region': region,
'samples': samples,
'source': None,
'weight': weight,
# use the exact same keys as in the moving case
'block_first': shard_station.block_first,
'block_last': shard_station.block_last,
'block_count': shard_station.block_count,
})
return ('changed', values)
return (None, None) # pragma: no cover
def update_area(self, session, areaid):
# Select all cells in this area and derive a bounding box for them
radio, mcc, mnc, lac = decode_cellarea(areaid)
load_fields = ('cellid', 'lat', 'lon', 'radius', 'region', 'last_seen',
'max_lat', 'max_lon', 'min_lat', 'min_lon')
shard = self.cell_model.shard_model(radio)
fields = [getattr(shard.__table__.c, f) for f in load_fields]
cells = session.execute(
select(fields)
.where(shard.__table__.c.radio == radio)
.where(shard.__table__.c.mcc == mcc)
.where(shard.__table__.c.mnc == mnc)
.where(shard.__table__.c.lac == lac)
.where(shard.__table__.c.lat.isnot(None))
.where(shard.__table__.c.lon.isnot(None))
).fetchall()
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
session.execute(
delete(self.area_table)
.where(self.area_table.c.areaid == areaid)
)
return
# Otherwise update the area entry based on all the cells
area = session.execute(
select([self.area_table.c.areaid,
self.area_table.c.modified,
self.area_table.c.lat,
self.area_table.c.lon,
self.area_table.c.radius,
self.area_table.c.region,
self.area_table.c.avg_cell_radius,
self.area_table.c.num_cells,
self.area_table.c.last_seen,
])
.where(self.area_table.c.areaid == areaid)
).fetchone()
cell_extremes = numpy.array([
(numpy.nan if cell.max_lat is None else cell.max_lat,
numpy.nan if cell.max_lon is None else cell.max_lon)
for cell in cells] + [
(numpy.nan if cell.min_lat is None else cell.min_lat,
numpy.nan if cell.min_lon is None else cell.min_lon)
for cell in cells
], dtype=numpy.double)
max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0)
min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0)
ctr_lat, ctr_lon = numpy.array(
[(c.lat, c.lon) for c in cells],
dtype=numpy.double).mean(axis=0)
ctr_lat = float(ctr_lat)
ctr_lon = float(ctr_lon)
radius = circle_radius(
ctr_lat, ctr_lon, max_lat, max_lon, min_lat, min_lon)
cell_radii = numpy.array([
(numpy.nan if cell.radius is None else cell.radius)
for cell in cells
], dtype=numpy.int32)
avg_cell_radius = int(round(numpy.nanmean(cell_radii)))
num_cells = len(cells)
region = self.region(ctr_lat, ctr_lon, mcc, cells)
last_seen = None
cell_last_seen = set([cell.last_seen for cell in cells
if cell.last_seen is not None])
if cell_last_seen:
last_seen = max(cell_last_seen)
if area is None:
session.execute(
self.area_table.insert(
mysql_on_duplicate='num_cells = num_cells' # no-op
).values(
areaid=areaid,
radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
radius=radius,
region=region,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
last_seen=last_seen,
)
)
else:
session.execute(
self.area_table.update()
.where(self.area_table.c.areaid == areaid)
.values(
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
radius=radius,
region=region,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
last_seen=last_seen,
)
)
