def to_geojson(regions): # pragma: no cover
features = []
for code, region in regions.items():
# calculate the maximum radius of each polygon
boundary = region.boundary
if isinstance(boundary, shapely.geometry.base.BaseMultipartGeometry):
boundaries = list(boundary.geoms)
else:
boundaries = [boundary]
radii = []
for boundary in boundaries:
# flip x/y aka lon/lat to lat/lon
boundary = numpy.fliplr(numpy.array(boundary))
ctr_lat, ctr_lon = geocalc.centroid(boundary)
radii.append(geocalc.max_distance(ctr_lat, ctr_lon, boundary))
radius = round(max(radii) / 1000.0) * 1000.0
features.append({
'type': 'Feature',
'properties': {
'alpha2': code,
'radius': radius,
},
'geometry': shapely.geometry.mapping(region),
})
def _sort(value):
return value['properties']['alpha2']
features = sorted(features, key=_sort)
lines = ',\n'.join([json.dumps(f, sort_keys=True, separators=(',', ':'))
for f in features])
return '{"type": "FeatureCollection", "features": [\n' + lines + '\n]}\n'
def to_geojson(regions): # pragma: no cover
features = []
for code, region in regions.items():
# calculate the maximum radius of each polygon
boundary = region.boundary
if isinstance(boundary, shapely.geometry.base.BaseMultipartGeometry):
boundaries = list(boundary.geoms)
else:
boundaries = [boundary]
radii = []
for boundary in boundaries:
# flip x/y aka lon/lat to lat/lon
boundary = numpy.fliplr(numpy.array(boundary))
ctr_lat, ctr_lon = geocalc.centroid(boundary)
radii.append(geocalc.max_distance(ctr_lat, ctr_lon, boundary))
radius = round(max(radii) / 1000.0) * 1000.0
features.append({
'type': 'Feature',
'properties': {
'alpha2': code,
'radius': radius,
},
'geometry': shapely.geometry.mapping(region),
})
def _sort(value):
return value['properties']['alpha2']
features = sorted(features, key=_sort)
lines = ',\n'.join([
json.dumps(f, sort_keys=True, separators=(',', ':')) for f in features
])
return '{"type": "FeatureCollection", "features": [\n' + lines + '\n]}\n'
def update(self, area_key):
# Select all cells in this area and derive a bounding box for them
cell_query = (self.cell_model.querykey(self.session, area_key)
.filter(self.cell_model.lat.isnot(None))
.filter(self.cell_model.lon.isnot(None)))
cells = cell_query.all()
area_query = self.cell_area_model.querykey(self.session, area_key)
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
area_query.delete()
else:
# Otherwise update the area entry based on all the cells
area = area_query.first()
points = [(c.lat, c.lon) for c in cells]
min_lat = min([c.min_lat for c in cells])
min_lon = min([c.min_lon for c in cells])
max_lat = max([c.max_lat for c in cells])
max_lon = max([c.max_lon for c in cells])
bbox_points = [(min_lat, min_lon),
(min_lat, max_lon),
(max_lat, min_lon),
(max_lat, max_lon)]
ctr = centroid(points)
rng = range_to_points(ctr, bbox_points)
# Switch units back to meters
ctr_lat = ctr[0]
ctr_lon = ctr[1]
rng = int(round(rng * 1000.0))
# Now create or update the area
num_cells = len(cells)
avg_cell_range = int(sum(
[cell.range for cell in cells]) / float(num_cells))
if area is None:
area = self.cell_area_model(
created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
range=rng,
avg_cell_range=avg_cell_range,
num_cells=num_cells,
**area_key.__dict__)
self.session.add(area)
else:
area.modified = self.utcnow
area.lat = ctr_lat
area.lon = ctr_lon
area.range = rng
area.avg_cell_range = avg_cell_range
area.num_cells = num_cells
def update_lac(self, radio, mcc, mnc, lac):
with self.db_session() as session:
# Select all the cells in this LAC that aren't the virtual
# cell itself, and derive a bounding box for them.
q = session.query(Cell).filter(Cell.radio == radio).filter(
Cell.mcc == mcc).filter(Cell.mnc == mnc).filter(
Cell.lac == lac).filter(Cell.cid != CELLID_LAC)
cells = q.all()
points = [(to_degrees(c.lat), to_degrees(c.lon)) for c in cells]
min_lat = to_degrees(min([c.min_lat for c in cells]))
min_lon = to_degrees(min([c.min_lon for c in cells]))
max_lat = to_degrees(max([c.max_lat for c in cells]))
max_lon = to_degrees(max([c.max_lon for c in cells]))
bbox_points = [(min_lat, min_lon), (min_lat, max_lon),
(max_lat, min_lon), (max_lat, max_lon)]
ctr = centroid(points)
rng = range_to_points(ctr, bbox_points)
# switch units back to DB preferred centimicrodegres angle
# and meters distance.
ctr_lat = from_degrees(ctr[0])
ctr_lon = from_degrees(ctr[1])
rng = int(round(rng * 1000.0))
# Now create or update the LAC virtual cell
q = session.query(Cell).filter(Cell.radio == radio).filter(
Cell.mcc == mcc).filter(Cell.mnc == mnc).filter(
Cell.lac == lac).filter(Cell.cid == CELLID_LAC)
lac = q.first()
if lac is None:
lac = Cell(radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
cid=CELLID_LAC,
lat=ctr_lat,
lon=ctr_lon,
range=rng)
else:
lac.new_measures = 0
lac.lat = ctr_lat
lac.lon = ctr_lon
lac.range = rng
session.commit()
def update(self, area_key):
# Select all cells in this area and derive a bounding box for them
cell_query = (self.cell_model.querykey(self.session, area_key).filter(
self.cell_model.lat.isnot(None)).filter(
self.cell_model.lon.isnot(None)))
cells = cell_query.all()
area_query = self.cell_area_model.querykey(self.session, area_key)
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
area_query.delete()
else:
# Otherwise update the area entry based on all the cells
area = area_query.first()
points = [(c.lat, c.lon) for c in cells]
min_lat = min([c.min_lat for c in cells])
min_lon = min([c.min_lon for c in cells])
max_lat = max([c.max_lat for c in cells])
max_lon = max([c.max_lon for c in cells])
bbox_points = [(min_lat, min_lon), (min_lat, max_lon),
(max_lat, min_lon), (max_lat, max_lon)]
ctr = centroid(points)
rng = range_to_points(ctr, bbox_points)
# Switch units back to meters
ctr_lat = ctr[0]
ctr_lon = ctr[1]
rng = int(round(rng * 1000.0))
# Now create or update the area
num_cells = len(cells)
avg_cell_range = int(
sum([cell.range for cell in cells]) / float(num_cells))
if area is None:
area = self.cell_area_model(created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
range=rng,
avg_cell_range=avg_cell_range,
num_cells=num_cells,
**area_key.__dict__)
self.session.add(area)
else:
area.modified = self.utcnow
area.lat = ctr_lat
area.lon = ctr_lon
area.range = rng
area.avg_cell_range = avg_cell_range
area.num_cells = num_cells
def station_values(self, station_key, shard_station, observations):
"""
Return two-tuple of status, value dict where status is one of:
`new`, `new_moving`, `moving`, `changed`.
"""
# cases:
# we always get a station key and observations
# 0. observations disagree
# 0.a. no shard station, return new_moving
# 0.b. shard station, return moving
# 1. no shard station
# 1.a. obs agree -> return new
# 2. shard station
# 2.a. obs disagree -> return moving
# 2.b. obs agree -> return changed
created = self.utcnow
values = self._base_station_values(station_key, observations)
obs_length = len(observations)
obs_positions = numpy.array([(obs.lat, obs.lon)
for obs in observations],
dtype=numpy.double)
obs_new_lat, obs_new_lon = centroid(obs_positions)
obs_max_lat, obs_max_lon = numpy.nanmax(obs_positions, axis=0)
obs_min_lat, obs_min_lon = numpy.nanmin(obs_positions, axis=0)
obs_box_dist = distance(obs_min_lat, obs_min_lon, obs_max_lat,
obs_max_lon)
if obs_box_dist > self.max_dist_meters:
# the new observations are already too far apart
if not shard_station:
values.update({
'created': created,
'block_first': self.today,
'block_last': self.today,
'block_count': 1,
})
return ('new_moving', values)
else:
block_count = shard_station.block_count or 0
values.update({
'lat':
None,
'lon':
None,
'max_lat':
None,
'min_lat':
None,
'max_lon':
None,
'min_lon':
None,
'radius':
None,
'region':
shard_station.region,
'samples':
None,
'source':
None,
'block_first':
shard_station.block_first or self.today,
'block_last':
self.today,
'block_count':
block_count + 1,
})
return ('moving', values)
if shard_station is None:
# totally new station, only agreeing observations
radius = circle_radius(obs_new_lat, obs_new_lon, obs_max_lat,
obs_max_lon, obs_min_lat, obs_min_lon)
values.update({
'created': created,
'lat': obs_new_lat,
'lon': obs_new_lon,
'max_lat': float(obs_max_lat),
'min_lat': float(obs_min_lat),
'max_lon': float(obs_max_lon),
'min_lon': float(obs_min_lon),
'radius': radius,
'region': GEOCODER.region(obs_new_lat, obs_new_lon),
'samples': obs_length,
'source': None,
})
return ('new', values)
else:
# shard_station + new observations
positions = numpy.append(obs_positions, [
(numpy.nan if shard_station.lat is None else shard_station.lat,
numpy.nan
if shard_station.lon is None else shard_station.lon),
(numpy.nan if shard_station.max_lat is None else
shard_station.max_lat, numpy.nan
if shard_station.max_lon is None else shard_station.max_lon),
(numpy.nan if shard_station.min_lat is None else
shard_station.min_lat, numpy.nan
if shard_station.min_lon is None else shard_station.min_lon),
],
axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
box_dist = distance(min_lat, min_lon, max_lat, max_lon)
if box_dist > self.max_dist_meters:
# shard_station + disagreeing observations
block_count = shard_station.block_count or 0
values.update({
'lat':
None,
'lon':
None,
'max_lat':
None,
'min_lat':
None,
'max_lon':
None,
'min_lon':
None,
'radius':
None,
'region':
shard_station.region,
'samples':
None,
'source':
None,
'block_first':
shard_station.block_first or self.today,
'block_last':
self.today,
'block_count':
block_count + 1,
})
return ('moving', values)
else:
# shard_station + agreeing observations
if shard_station.lat is None or shard_station.lon is None:
old_weight = 0
else:
old_weight = min((shard_station.samples or 0),
self.MAX_OLD_OBSERVATIONS)
new_lat = ((obs_new_lat * obs_length +
(shard_station.lat or 0.0) * old_weight) /
(obs_length + old_weight))
new_lon = ((obs_new_lon * obs_length +
(shard_station.lon or 0.0) * old_weight) /
(obs_length + old_weight))
samples = (shard_station.samples or 0) + obs_length
radius = circle_radius(new_lat, new_lon, max_lat, max_lon,
min_lat, min_lon)
region = shard_station.region
if (region
and not GEOCODER.in_region(new_lat, new_lon, region)):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(new_lat, new_lon)
values.update({
'lat': new_lat,
'lon': new_lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'radius': radius,
'region': region,
'samples': samples,
'source': None,
# use the exact same keys as in the moving case
'block_first': shard_station.block_first,
'block_last': shard_station.block_last,
'block_count': shard_station.block_count,
})
return ('changed', values)
return (None, None) # pragma: no cover
def update_lac(self, radio, mcc, mnc, lac):
with self.db_session() as session:
# Select all the cells in this LAC that aren't the virtual
# cell itself, and derive a bounding box for them.
q = session.query(Cell).filter(
Cell.radio == radio).filter(
Cell.mcc == mcc).filter(
Cell.mnc == mnc).filter(
Cell.lac == lac).filter(
Cell.cid != CELLID_LAC).filter(
Cell.new_measures == 0).filter(
Cell.lat.isnot(None)).filter(
Cell.lon.isnot(None))
cells = q.all()
if len(cells) == 0:
return
points = [(to_degrees(c.lat),
to_degrees(c.lon)) for c in cells]
min_lat = to_degrees(min([c.min_lat for c in cells]))
min_lon = to_degrees(min([c.min_lon for c in cells]))
max_lat = to_degrees(max([c.max_lat for c in cells]))
max_lon = to_degrees(max([c.max_lon for c in cells]))
bbox_points = [(min_lat, min_lon),
(min_lat, max_lon),
(max_lat, min_lon),
(max_lat, max_lon)]
ctr = centroid(points)
rng = range_to_points(ctr, bbox_points)
# switch units back to DB preferred centimicrodegres angle
# and meters distance.
ctr_lat = from_degrees(ctr[0])
ctr_lon = from_degrees(ctr[1])
rng = int(round(rng * 1000.0))
# Now create or update the LAC virtual cell
q = session.query(Cell).filter(
Cell.radio == radio).filter(
Cell.mcc == mcc).filter(
Cell.mnc == mnc).filter(
Cell.lac == lac).filter(
Cell.cid == CELLID_LAC)
lac = q.first()
if lac is None:
lac = Cell(radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
cid=CELLID_LAC,
lat=ctr_lat,
lon=ctr_lon,
range=rng)
else:
lac.new_measures = 0
lac.lat = ctr_lat
lac.lon = ctr_lon
lac.range = rng
session.commit()
def update_area(self, areaid):
# Select all cells in this area and derive a bounding box for them
radio, mcc, mnc, lac = decode_cellarea(areaid)
load_fields = ('lat', 'lon', 'radius', 'region',
'max_lat', 'max_lon', 'min_lat', 'min_lon')
shard = self.cell_model.shard_model(radio)
cells = (self.session.query(shard)
.options(load_only(*load_fields))
.filter(shard.radio == radio)
.filter(shard.mcc == mcc)
.filter(shard.mnc == mnc)
.filter(shard.lac == lac)
.filter(shard.lat.isnot(None))
.filter(shard.lon.isnot(None))).all()
area_query = (self.session.query(self.area_model)
.filter(self.area_model.areaid == areaid))
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
area_query.delete()
else:
# Otherwise update the area entry based on all the cells
area = area_query.first()
cell_extremes = numpy.array([
(numpy.nan if cell.max_lat is None else cell.max_lat,
numpy.nan if cell.max_lon is None else cell.max_lon)
for cell in cells] + [
(numpy.nan if cell.min_lat is None else cell.min_lat,
numpy.nan if cell.min_lon is None else cell.min_lon)
for cell in cells
], dtype=numpy.double)
max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0)
min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0)
ctr_lat, ctr_lon = centroid(
numpy.array([(c.lat, c.lon) for c in cells],
dtype=numpy.double))
radius = circle_radius(
ctr_lat, ctr_lon, max_lat, max_lon, min_lat, min_lon)
cell_radii = numpy.array([
(numpy.nan if cell.radius is None else cell.radius)
for cell in cells
], dtype=numpy.int32)
avg_cell_radius = int(round(numpy.nanmean(cell_radii)))
num_cells = len(cells)
region = self.region(ctr_lat, ctr_lon, mcc, cells)
if area is None:
stmt = self.area_model.__table__.insert(
mysql_on_duplicate='num_cells = num_cells' # no-op
).values(
areaid=areaid,
radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
radius=radius,
region=region,
avg_cell_radius=avg_cell_radius,
num_cells=num_cells,
)
self.session.execute(stmt)
else:
area.modified = self.utcnow
area.lat = ctr_lat
area.lon = ctr_lon
area.radius = radius
area.region = region
area.avg_cell_radius = avg_cell_radius
area.num_cells = num_cells
def station_values(self, station_key, shard_station, observations):
"""
Return two-tuple of status, value dict where status is one of:
`new`, `new_moving`, `moving`, `changed`.
"""
# cases:
# we always get a station key and observations
# 0. observations disagree
# 0.a. no shard station, return new_moving
# 0.b. shard station, return moving
# 1. no shard station
# 1.a. obs agree -> return new
# 2. shard station
# 2.a. obs disagree -> return moving
# 2.b. obs agree -> return changed
created = self.utcnow
values = {
'mac': station_key,
'modified': self.utcnow,
}
obs_length = len(observations)
obs_positions = numpy.array(
[(obs.lat, obs.lon) for obs in observations],
dtype=numpy.double)
obs_new_lat, obs_new_lon = centroid(obs_positions)
obs_max_lat, obs_max_lon = numpy.nanmax(obs_positions, axis=0)
obs_min_lat, obs_min_lon = numpy.nanmin(obs_positions, axis=0)
obs_box_dist = distance(obs_min_lat, obs_min_lon,
obs_max_lat, obs_max_lon)
if obs_box_dist > self.max_dist_meters:
# the new observations are already too far apart
if not shard_station:
values.update({
'created': created,
'block_first': self.today,
'block_last': self.today,
'block_count': 1,
})
return ('new_moving', values)
else:
block_count = shard_station.block_count or 0
values.update({
'lat': None,
'lon': None,
'max_lat': None,
'min_lat': None,
'max_lon': None,
'min_lon': None,
'country': shard_station.country,
'radius': None,
'samples': None,
'source': None,
'block_last': self.today,
'block_count': block_count + 1,
})
return ('moving', values)
if shard_station is None:
# totally new station, only agreeing observations
radius = circle_radius(
obs_new_lat, obs_new_lon,
obs_max_lat, obs_max_lon, obs_min_lat, obs_min_lon)
values.update({
'created': created,
'lat': obs_new_lat,
'lon': obs_new_lon,
'max_lat': float(obs_max_lat),
'min_lat': float(obs_min_lat),
'max_lon': float(obs_max_lon),
'min_lon': float(obs_min_lon),
'country': country_for_location(obs_new_lat, obs_new_lon),
'radius': radius,
'samples': obs_length,
'source': None,
})
return ('new', values)
else:
# shard_station + new observations
positions = numpy.append(obs_positions, [
(numpy.nan if shard_station.lat is None
else shard_station.lat,
numpy.nan if shard_station.lon is None
else shard_station.lon),
(numpy.nan if shard_station.max_lat is None
else shard_station.max_lat,
numpy.nan if shard_station.max_lon is None
else shard_station.max_lon),
(numpy.nan if shard_station.min_lat is None
else shard_station.min_lat,
numpy.nan if shard_station.min_lon is None
else shard_station.min_lon),
], axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
box_dist = distance(min_lat, min_lon, max_lat, max_lon)
if box_dist > self.max_dist_meters:
# shard_station + disagreeing observations
block_count = shard_station.block_count or 0
values.update({
'lat': None,
'lon': None,
'max_lat': None,
'min_lat': None,
'max_lon': None,
'min_lon': None,
'country': shard_station.country,
'radius': None,
'samples': None,
'source': None,
'block_last': self.today,
'block_count': block_count + 1,
})
return ('moving', values)
else:
# shard_station + agreeing observations
if shard_station.lat is None or shard_station.lon is None:
old_weight = 0
else:
old_weight = min((shard_station.samples or 0),
self.MAX_OLD_OBSERVATIONS)
new_lat = ((obs_new_lat * obs_length +
(shard_station.lat or 0.0) * old_weight) /
(obs_length + old_weight))
new_lon = ((obs_new_lon * obs_length +
(shard_station.lon or 0.0) * old_weight) /
(obs_length + old_weight))
samples = (shard_station.samples or 0) + obs_length
radius = circle_radius(
new_lat, new_lon, max_lat, max_lon, min_lat, min_lon)
country = shard_station.country
if (country and not country_matches_location(
new_lat, new_lon, country)):
# reset country if it no longer matches
country = None
if not country:
country = country_for_location(new_lat, new_lon)
values.update({
'lat': new_lat,
'lon': new_lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'country': country,
'radius': radius,
'samples': samples,
'source': None,
# use the exact same keys as in the moving case
'block_last': shard_station.block_last,
'block_count': shard_station.block_count,
})
return ('changed', values)
return (None, None) # pragma: no cover
def new_station_values(self, station, station_key,
first_blocked, observations):
# This function returns a 3-tuple, the first element is True,
# if the station was found to be moving.
# The second element is either None or a dict of values,
# if the station is new and should result in a table insert
# The third element is either None or a dict of values
# if the station did exist and should be updated
obs_length = len(observations)
obs_positions = numpy.array(
[(obs.lat, obs.lon) for obs in observations],
dtype=numpy.double)
obs_lat, obs_lon = centroid(obs_positions)
values = {
'modified': self.utcnow,
}
values.update(station_key.__dict__)
if self.station_type == 'cell':
# pass on extra psc column which is not actually part
# of the stations hash key
values['psc'] = observations[-1].psc
created = self.utcnow
if station is None:
if first_blocked:
# if the station did previously exist, retain at least the
# time it was first put on a blocklist as the creation date
created = first_blocked
values.update({
'created': created,
'range': 0,
'total_measures': 0,
})
if (station is not None and
station.lat is not None and station.lon is not None):
obs_positions = numpy.append(obs_positions, [
(station.lat, station.lon),
(numpy.nan if station.max_lat is None else station.max_lat,
numpy.nan if station.max_lon is None else station.max_lon),
(numpy.nan if station.min_lat is None else station.min_lat,
numpy.nan if station.min_lon is None else station.min_lon),
], axis=0)
existing_station = True
else:
values['lat'] = obs_lat
values['lon'] = obs_lon
existing_station = False
max_lat, max_lon = numpy.nanmax(obs_positions, axis=0)
min_lat, min_lon = numpy.nanmin(obs_positions, axis=0)
# calculate sphere-distance from opposite corners of
# bounding box containing current location estimate
# and new observations; if too big, station is moving
box_dist = distance(min_lat, min_lon, max_lat, max_lon)
# TODO: If we get a too large box_dist, we should not create
# a new station record with the impossibly big distance,
# so moving the box_dist > self.max_dist_meters here
if existing_station:
if box_dist > self.max_dist_meters:
# Signal a moving station and return early without updating
# the station since it will be deleted by caller momentarily
return (True, None, None)
# limit the maximum weight of the old station estimate
old_weight = min(station.total_measures,
self.MAX_OLD_OBSERVATIONS)
new_weight = old_weight + obs_length
values['lat'] = ((station.lat * old_weight) +
(obs_lat * obs_length)) / new_weight
values['lon'] = ((station.lon * old_weight) +
(obs_lon * obs_length)) / new_weight
# increase total counter
if station is not None:
values['total_measures'] = station.total_measures + obs_length
else:
values['total_measures'] = obs_length
# update max/min lat/lon columns
values['min_lat'] = float(min_lat)
values['min_lon'] = float(min_lon)
values['max_lat'] = float(max_lat)
values['max_lon'] = float(max_lon)
# give radio-range estimate between extreme values and centroid
values['range'] = circle_radius(
values['lat'], values['lon'],
max_lat, max_lon, min_lat, min_lon)
if station is None:
# return new values
return (False, values, None)
else:
# return updated values, remove station from session
self.session.expunge(station)
return (False, None, values)
def update_area(self, area_key):
# Select all cells in this area and derive a bounding box for them
cell_query = (self.cell_model.querykey(self.session, area_key)
.filter(self.cell_model.lat.isnot(None))
.filter(self.cell_model.lon.isnot(None)))
cells = cell_query.all()
area_query = self.area_model.querykey(self.session, area_key)
if len(cells) == 0:
# If there are no more underlying cells, delete the area entry
area_query.delete()
else:
# Otherwise update the area entry based on all the cells
area = area_query.first()
cell_extremes = numpy.array([
(numpy.nan if cell.max_lat is None else cell.max_lat,
numpy.nan if cell.max_lon is None else cell.max_lon)
for cell in cells] + [
(numpy.nan if cell.min_lat is None else cell.min_lat,
numpy.nan if cell.min_lon is None else cell.min_lon)
for cell in cells
], dtype=numpy.double)
max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0)
min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0)
ctr_lat, ctr_lon = centroid(
numpy.array([(c.lat, c.lon) for c in cells],
dtype=numpy.double))
radius = circle_radius(
ctr_lat, ctr_lon,
max_lat, max_lon, min_lat, min_lon)
# Now create or update the area
cell_ranges = numpy.array([
(numpy.nan if cell.range is None else cell.range)
for cell in cells
], dtype=numpy.int32)
avg_cell_range = int(round(numpy.nanmean(cell_ranges)))
num_cells = len(cells)
if area is None:
stmt = self.area_model.__table__.insert(
mysql_on_duplicate='num_cells = num_cells' # no-op
).values(
created=self.utcnow,
modified=self.utcnow,
lat=ctr_lat,
lon=ctr_lon,
range=radius,
avg_cell_range=avg_cell_range,
num_cells=num_cells,
**area_key.__dict__
)
self.session.execute(stmt)
else:
area.modified = self.utcnow
area.lat = ctr_lat
area.lon = ctr_lon
area.range = radius
area.avg_cell_range = avg_cell_range
area.num_cells = num_cells
def to_geojson(regions): # pragma: no cover
features = []
features_buffered = []
for code, region in regions.items():
# calculate the maximum radius of each polygon
boundary = region.boundary
if isinstance(boundary, shapely.geometry.base.BaseMultipartGeometry):
boundaries = list(boundary.geoms)
else:
boundaries = [boundary]
radii = []
for boundary in boundaries:
# flip x/y aka lon/lat to lat/lon
boundary = numpy.fliplr(numpy.array(boundary))
ctr_lat, ctr_lon = geocalc.centroid(boundary)
radii.append(geocalc.max_distance(ctr_lat, ctr_lon, boundary))
radius = round(max(radii) / 1000.0) * 1000.0
features.append({
'type': 'Feature',
'properties': {
'alpha2': code,
'radius': radius,
},
'geometry': shapely.geometry.mapping(region),
})
# Build up region buffers, to create shapes that include all of
# the coastal areas and boundaries of the regions and anywhere
# a cell signal could still be recorded. The value is in decimal
# degrees (1.0 == ~100km) but calculations don't take projection
# / WSG84 into account.
# After buffering remove any parts that crosses the -180.0/+180.0
# longitude boundary to the east or west.
buffered = (region.buffer(0.5)
.simplify(0.02)
.difference(DATELINE_EAST)
.difference(DATELINE_WEST))
features_buffered.append({
'type': 'Feature',
'properties': {
'alpha2': code,
},
'geometry': shapely.geometry.mapping(buffered),
})
def _sort(value):
return value['properties']['alpha2']
def _to_collection(features):
# put each region on a single line to get a diffable output
lines = ',\n'.join([json.dumps(feature,
sort_keys=True,
separators=(',', ':'))
for feature in sorted(features, key=_sort)])
return ('{"type": "FeatureCollection", '
'"features": [\n' + lines + '\n]}\n')
return (_to_collection(features), _to_collection(features_buffered))
def update_lac(self, radio, mcc, mnc, lac):
try:
utcnow = util.utcnow()
with self.db_session() as session:
# Select all the cells in this LAC that aren't the virtual
# cell itself, and derive a bounding box for them.
cell_query = session.query(Cell).filter(
Cell.radio == radio).filter(
Cell.mcc == mcc).filter(
Cell.mnc == mnc).filter(
Cell.lac == lac).filter(
Cell.cid != CELLID_LAC).filter(
Cell.lat.isnot(None)).filter(
Cell.lon.isnot(None))
cells = cell_query.all()
lac_query = session.query(Cell).filter(
Cell.radio == radio).filter(
Cell.mcc == mcc).filter(
Cell.mnc == mnc).filter(
Cell.lac == lac).filter(
Cell.cid == CELLID_LAC)
if len(cells) == 0:
# If there are no more underlying cells, delete the lac entry
lac_query.delete()
else:
# Otherwise update the lac entry based on all the cells
lac_obj = lac_query.first()
points = [(c.lat, c.lon) for c in cells]
min_lat = min([c.min_lat for c in cells])
min_lon = min([c.min_lon for c in cells])
max_lat = max([c.max_lat for c in cells])
max_lon = max([c.max_lon for c in cells])
bbox_points = [(min_lat, min_lon),
(min_lat, max_lon),
(max_lat, min_lon),
(max_lat, max_lon)]
ctr = centroid(points)
rng = range_to_points(ctr, bbox_points)
# Switch units back to DB preferred centimicrodegres angle
# and meters distance.
ctr_lat = ctr[0]
ctr_lon = ctr[1]
rng = int(round(rng * 1000.0))
# Now create or update the LAC virtual cell
if lac_obj is None:
lac_obj = Cell(radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
cid=CELLID_LAC,
lat=ctr_lat,
lon=ctr_lon,
created=utcnow,
modified=utcnow,
range=rng,
total_measures=len(cells))
session.add(lac_obj)
else:
lac_obj.total_measures = len(cells)
lac_obj.lat = ctr_lat
lac_obj.lon = ctr_lon
lac_obj.modified = utcnow
lac_obj.range = rng
session.commit()
except Exception as exc: # pragma: no cover
self.heka_client.raven('error')
raise self.retry(exc=exc)
def update_lac(self,
radio,
mcc,
mnc,
lac,
cell_model_key='cell',
cell_area_model_key='cell_area'):
utcnow = util.utcnow()
with self.db_session() as session:
# Select all the cells in this LAC that aren't the virtual
# cell itself, and derive a bounding box for them.
cell_model = CELL_MODEL_KEYS[cell_model_key]
cell_query = (session.query(cell_model).filter(
cell_model.radio == radio).filter(cell_model.mcc == mcc).filter(
cell_model.mnc == mnc).filter(cell_model.lac == lac).filter(
cell_model.lat.isnot(None)).filter(
cell_model.lon.isnot(None)))
cells = cell_query.all()
cell_area_model = CELL_MODEL_KEYS[cell_area_model_key]
lac_query = (session.query(cell_area_model).filter(
cell_area_model.radio == radio).filter(
cell_area_model.mcc == mcc).filter(
cell_area_model.mnc == mnc).filter(
cell_area_model.lac == lac))
if len(cells) == 0:
# If there are no more underlying cells, delete the lac entry
lac_query.delete()
else:
# Otherwise update the lac entry based on all the cells
lac_obj = lac_query.first()
points = [(c.lat, c.lon) for c in cells]
min_lat = min([c.min_lat for c in cells])
min_lon = min([c.min_lon for c in cells])
max_lat = max([c.max_lat for c in cells])
max_lon = max([c.max_lon for c in cells])
bbox_points = [(min_lat, min_lon), (min_lat, max_lon),
(max_lat, min_lon), (max_lat, max_lon)]
ctr = centroid(points)
rng = range_to_points(ctr, bbox_points)
# Switch units back to DB preferred centimicrodegres angle
# and meters distance.
ctr_lat = ctr[0]
ctr_lon = ctr[1]
rng = int(round(rng * 1000.0))
# Now create or update the LAC virtual cell
num_cells = len(cells)
avg_cell_range = int(
sum([cell.range for cell in cells]) / float(num_cells))
if lac_obj is None:
lac_obj = cell_area_model(
created=utcnow,
modified=utcnow,
radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
lat=ctr_lat,
lon=ctr_lon,
range=rng,
avg_cell_range=avg_cell_range,
num_cells=num_cells,
)
session.add(lac_obj)
else:
lac_obj.modified = utcnow
lac_obj.lat = ctr_lat
lac_obj.lon = ctr_lon
lac_obj.range = rng
lac_obj.avg_cell_range = avg_cell_range
lac_obj.num_cells = num_cells
session.commit()
def update_lac(self, radio, mcc, mnc, lac):
try:
utcnow = util.utcnow()
with self.db_session() as session:
# Select all the cells in this LAC that aren't the virtual
# cell itself, and derive a bounding box for them.
cell_query = session.query(Cell).filter(
Cell.radio == radio).filter(Cell.mcc == mcc).filter(
Cell.mnc == mnc).filter(
Cell.lac == lac).filter(Cell.cid != CELLID_LAC).filter(
Cell.lat.isnot(None)).filter(Cell.lon.isnot(None))
cells = cell_query.all()
lac_query = session.query(Cell).filter(Cell.radio == radio).filter(
Cell.mcc == mcc).filter(Cell.mnc == mnc).filter(
Cell.lac == lac).filter(Cell.cid == CELLID_LAC)
if len(cells) == 0:
# If there are no more underlying cells, delete the lac entry
lac_query.delete()
else:
# Otherwise update the lac entry based on all the cells
lac_obj = lac_query.first()
points = [(c.lat, c.lon) for c in cells]
min_lat = min([c.min_lat for c in cells])
min_lon = min([c.min_lon for c in cells])
max_lat = max([c.max_lat for c in cells])
max_lon = max([c.max_lon for c in cells])
bbox_points = [(min_lat, min_lon), (min_lat, max_lon),
(max_lat, min_lon), (max_lat, max_lon)]
ctr = centroid(points)
rng = range_to_points(ctr, bbox_points)
# Switch units back to DB preferred centimicrodegres angle
# and meters distance.
ctr_lat = ctr[0]
ctr_lon = ctr[1]
rng = int(round(rng * 1000.0))
# Now create or update the LAC virtual cell
if lac_obj is None:
lac_obj = Cell(radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
cid=CELLID_LAC,
lat=ctr_lat,
lon=ctr_lon,
created=utcnow,
modified=utcnow,
range=rng,
total_measures=len(cells))
session.add(lac_obj)
else:
lac_obj.total_measures = len(cells)
lac_obj.lat = ctr_lat
lac_obj.lon = ctr_lon
lac_obj.modified = utcnow
lac_obj.range = rng
session.commit()
except Exception as exc: # pragma: no cover
self.heka_client.raven('error')
raise self.retry(exc=exc)
