def aggregate_obs(self): positions = numpy.array( [(obs.lat, obs.lon) for obs in self.observations], dtype=numpy.double) max_lat, max_lon = positions.max(axis=0) min_lat, min_lon = positions.min(axis=0) box_distance = distance(min_lat, min_lon, max_lat, max_lon) if box_distance > self.MAX_DIST_METERS: return None weights = numpy.array( [obs.weight for obs in self.observations], dtype=numpy.double) lat, lon = numpy.average(positions, axis=0, weights=weights) lat = float(lat) lon = float(lon) radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon) region = GEOCODER.region(lat, lon) samples, weight = self.bounded_samples_weight( len(self.observations), float(weights.sum())) return { 'positions': positions, 'weights': weights, 'lat': lat, 'lon': lon, 'max_lat': float(max_lat), 'min_lat': float(min_lat), 'max_lon': float(max_lon), 'min_lon': float(min_lon), 'radius': radius, 'region': region, 'samples': samples, 'weight': weight, }
def _update_area(bind, shard_id, areaid, circle_radius): radio, mcc, mnc, lac = CELLAREA_STRUCT.unpack(codecs.decode(areaid, 'hex')) row = bind.execute( sa.text( stmt_select_area.format(id=shard_id, radio=radio, mcc=mcc, mnc=mnc, lac=lac))).fetchone() num_cells = int(row.num_cells) if num_cells == 0: op.execute( sa.text( stmt_delete_area.format(radio=radio, mcc=mcc, mnc=mnc, lac=lac))) else: radius = circle_radius(float(row.lat), float(row.lon), float(row.max_lat), float(row.max_lon), float(row.min_lat), float(row.min_lon)) avg_cell_radius = int(round(row.avg_cell_radius)) op.execute( sa.text( stmt_update_area.format( radio=radio, mcc=mcc, mnc=mnc, lac=lac, lat=float(row.lat), lon=float(row.lon), radius=radius, avg_cell_radius=avg_cell_radius, num_cells=num_cells, )))
def aggregate_station_obs(self): station = self.station obs_data = self.obs_data def get_nan(name): value = getattr(station, name, None) return numpy.nan if value is None else value positions = numpy.append(obs_data['positions'], [ (get_nan('lat'), get_nan('lon')), (get_nan('max_lat'), get_nan('max_lon')), (get_nan('min_lat'), get_nan('min_lon')), ], axis=0) max_lat, max_lon = numpy.nanmax(positions, axis=0) min_lat, min_lon = numpy.nanmin(positions, axis=0) if station.lat is None or station.lon is None: old_weight = 0.0 else: old_weight = min((station.weight or 0.0), self.MAX_OLD_WEIGHT) lat = ((obs_data['lat'] * obs_data['weight'] + (station.lat or 0.0) * old_weight) / (obs_data['weight'] + old_weight)) lon = ((obs_data['lon'] * obs_data['weight'] + (station.lon or 0.0) * old_weight) / (obs_data['weight'] + old_weight)) radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon) region = station.region if (region and not GEOCODER.in_region(lat, lon, region)): # reset region if it no longer matches region = None if not region: region = GEOCODER.region(lat, lon) samples, weight = self.bounded_samples_weight( (station.samples or 0) + obs_data['samples'], (station.weight or 0.0) + obs_data['weight']) return { 'lat': lat, 'lon': lon, 'max_lat': float(max_lat), 'min_lat': float(min_lat), 'max_lon': float(max_lon), 'min_lon': float(min_lon), 'radius': radius, 'region': region, 'samples': samples, 'weight': weight, }
def _update_area(bind, shard_id, areaid, circle_radius): radio, mcc, mnc, lac = CELLAREA_STRUCT.unpack(codecs.decode(areaid, 'hex')) row = bind.execute(sa.text(stmt_select_area.format( id=shard_id, radio=radio, mcc=mcc, mnc=mnc, lac=lac))).fetchone() num_cells = int(row.num_cells) if num_cells == 0: op.execute(sa.text(stmt_delete_area.format( radio=radio, mcc=mcc, mnc=mnc, lac=lac))) else: radius = circle_radius( float(row.lat), float(row.lon), float(row.max_lat), float(row.max_lon), float(row.min_lat), float(row.min_lon)) avg_cell_radius = int(round(row.avg_cell_radius)) op.execute(sa.text(stmt_update_area.format( radio=radio, mcc=mcc, mnc=mnc, lac=lac, lat=float(row.lat), lon=float(row.lon), radius=radius, avg_cell_radius=avg_cell_radius, num_cells=num_cells, )))
def aggregate_obs(self): positions = numpy.array([(obs.lat, obs.lon) for obs in self.observations], dtype=numpy.double) max_lat, max_lon = positions.max(axis=0) min_lat, min_lon = positions.min(axis=0) box_distance = distance(min_lat, min_lon, max_lat, max_lon) if box_distance > self.MAX_DIST_METERS: return None weights = numpy.array([obs.weight for obs in self.observations], dtype=numpy.double) lat, lon = numpy.average(positions, axis=0, weights=weights) lat = float(lat) lon = float(lon) radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon) region = GEOCODER.region(lat, lon) samples, weight = self.bounded_samples_weight(len(self.observations), float(weights.sum())) return { 'positions': positions, 'weights': weights, 'lat': lat, 'lon': lon, 'max_lat': float(max_lat), 'min_lat': float(min_lat), 'max_lon': float(max_lon), 'min_lon': float(min_lon), 'radius': radius, 'region': region, 'samples': samples, 'weight': weight, }
def station_values(self, station_key, shard_station, observations): """ Return two-tuple of status, value dict where status is one of: `new`, `new_moving`, `moving`, `changed`. """ # cases: # we always get a station key and observations # 0. observations disagree # 0.a. no shard station, return new_moving # 0.b. shard station, return moving # 1. no shard station # 1.a. obs agree -> return new # 2. shard station # 2.a. obs disagree -> return moving # 2.b. obs agree -> return changed created = self.utcnow values = self._base_station_values(station_key, observations) obs_length = len(observations) obs_positions = numpy.array([(obs.lat, obs.lon) for obs in observations], dtype=numpy.double) obs_new_lat, obs_new_lon = centroid(obs_positions) obs_max_lat, obs_max_lon = numpy.nanmax(obs_positions, axis=0) obs_min_lat, obs_min_lon = numpy.nanmin(obs_positions, axis=0) obs_box_dist = distance(obs_min_lat, obs_min_lon, obs_max_lat, obs_max_lon) if obs_box_dist > self.max_dist_meters: # the new observations are already too far apart if not shard_station: values.update({ 'created': created, 'block_first': self.today, 'block_last': self.today, 'block_count': 1, }) return ('new_moving', values) else: block_count = shard_station.block_count or 0 values.update({ 'lat': None, 'lon': None, 'max_lat': None, 'min_lat': None, 'max_lon': None, 'min_lon': None, 'radius': None, 'region': shard_station.region, 'samples': None, 'source': None, 'block_first': shard_station.block_first or self.today, 'block_last': self.today, 'block_count': block_count + 1, }) return ('moving', values) if shard_station is None: # totally new station, only agreeing observations radius = circle_radius(obs_new_lat, obs_new_lon, obs_max_lat, obs_max_lon, obs_min_lat, obs_min_lon) values.update({ 'created': created, 'lat': obs_new_lat, 'lon': obs_new_lon, 'max_lat': float(obs_max_lat), 'min_lat': float(obs_min_lat), 'max_lon': float(obs_max_lon), 'min_lon': float(obs_min_lon), 'radius': radius, 'region': GEOCODER.region(obs_new_lat, obs_new_lon), 'samples': obs_length, 'source': None, }) return ('new', values) else: # shard_station + new observations positions = numpy.append(obs_positions, [ (numpy.nan if shard_station.lat is None else shard_station.lat, numpy.nan if shard_station.lon is None else shard_station.lon), (numpy.nan if shard_station.max_lat is None else shard_station.max_lat, numpy.nan if shard_station.max_lon is None else shard_station.max_lon), (numpy.nan if shard_station.min_lat is None else shard_station.min_lat, numpy.nan if shard_station.min_lon is None else shard_station.min_lon), ], axis=0) max_lat, max_lon = numpy.nanmax(positions, axis=0) min_lat, min_lon = numpy.nanmin(positions, axis=0) box_dist = distance(min_lat, min_lon, max_lat, max_lon) if box_dist > self.max_dist_meters: # shard_station + disagreeing observations block_count = shard_station.block_count or 0 values.update({ 'lat': None, 'lon': None, 'max_lat': None, 'min_lat': None, 'max_lon': None, 'min_lon': None, 'radius': None, 'region': shard_station.region, 'samples': None, 'source': None, 'block_first': shard_station.block_first or self.today, 'block_last': self.today, 'block_count': block_count + 1, }) return ('moving', values) else: # shard_station + agreeing observations if shard_station.lat is None or shard_station.lon is None: old_weight = 0 else: old_weight = min((shard_station.samples or 0), self.MAX_OLD_OBSERVATIONS) new_lat = ((obs_new_lat * obs_length + (shard_station.lat or 0.0) * old_weight) / (obs_length + old_weight)) new_lon = ((obs_new_lon * obs_length + (shard_station.lon or 0.0) * old_weight) / (obs_length + old_weight)) samples = (shard_station.samples or 0) + obs_length radius = circle_radius(new_lat, new_lon, max_lat, max_lon, min_lat, min_lon) region = shard_station.region if (region and not GEOCODER.in_region(new_lat, new_lon, region)): # reset region if it no longer matches region = None if not region: region = GEOCODER.region(new_lat, new_lon) values.update({ 'lat': new_lat, 'lon': new_lon, 'max_lat': float(max_lat), 'min_lat': float(min_lat), 'max_lon': float(max_lon), 'min_lon': float(min_lon), 'radius': radius, 'region': region, 'samples': samples, 'source': None, # use the exact same keys as in the moving case 'block_first': shard_station.block_first, 'block_last': shard_station.block_last, 'block_count': shard_station.block_count, }) return ('changed', values) return (None, None) # pragma: no cover
def update_area(self, areaid): # Select all cells in this area and derive a bounding box for them radio, mcc, mnc, lac = decode_cellarea(areaid) load_fields = ('lat', 'lon', 'radius', 'region', 'max_lat', 'max_lon', 'min_lat', 'min_lon') shard = self.cell_model.shard_model(radio) cells = (self.session.query(shard) .options(load_only(*load_fields)) .filter(shard.radio == radio) .filter(shard.mcc == mcc) .filter(shard.mnc == mnc) .filter(shard.lac == lac) .filter(shard.lat.isnot(None)) .filter(shard.lon.isnot(None))).all() area_query = (self.session.query(self.area_model) .filter(self.area_model.areaid == areaid)) if len(cells) == 0: # If there are no more underlying cells, delete the area entry area_query.delete() else: # Otherwise update the area entry based on all the cells area = area_query.first() cell_extremes = numpy.array([ (numpy.nan if cell.max_lat is None else cell.max_lat, numpy.nan if cell.max_lon is None else cell.max_lon) for cell in cells] + [ (numpy.nan if cell.min_lat is None else cell.min_lat, numpy.nan if cell.min_lon is None else cell.min_lon) for cell in cells ], dtype=numpy.double) max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0) min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0) ctr_lat, ctr_lon = centroid( numpy.array([(c.lat, c.lon) for c in cells], dtype=numpy.double)) radius = circle_radius( ctr_lat, ctr_lon, max_lat, max_lon, min_lat, min_lon) cell_radii = numpy.array([ (numpy.nan if cell.radius is None else cell.radius) for cell in cells ], dtype=numpy.int32) avg_cell_radius = int(round(numpy.nanmean(cell_radii))) num_cells = len(cells) region = self.region(ctr_lat, ctr_lon, mcc, cells) if area is None: stmt = self.area_model.__table__.insert( mysql_on_duplicate='num_cells = num_cells' # no-op ).values( areaid=areaid, radio=radio, mcc=mcc, mnc=mnc, lac=lac, created=self.utcnow, modified=self.utcnow, lat=ctr_lat, lon=ctr_lon, radius=radius, region=region, avg_cell_radius=avg_cell_radius, num_cells=num_cells, ) self.session.execute(stmt) else: area.modified = self.utcnow area.lat = ctr_lat area.lon = ctr_lon area.radius = radius area.region = region area.avg_cell_radius = avg_cell_radius area.num_cells = num_cells
def station_values(self, station_key, shard_station, observations): """ Return two-tuple of status, value dict where status is one of: `new`, `new_moving`, `moving`, `changed`. """ # cases: # we always get a station key and observations # 0. observations disagree # 0.a. no shard station, return new_moving # 0.b. shard station, return moving # 1. no shard station # 1.a. obs agree -> return new # 2. shard station # 2.a. obs disagree -> return moving # 2.b. obs agree -> return changed created = self.utcnow values = { 'mac': station_key, 'modified': self.utcnow, } obs_length = len(observations) obs_positions = numpy.array( [(obs.lat, obs.lon) for obs in observations], dtype=numpy.double) obs_new_lat, obs_new_lon = centroid(obs_positions) obs_max_lat, obs_max_lon = numpy.nanmax(obs_positions, axis=0) obs_min_lat, obs_min_lon = numpy.nanmin(obs_positions, axis=0) obs_box_dist = distance(obs_min_lat, obs_min_lon, obs_max_lat, obs_max_lon) if obs_box_dist > self.max_dist_meters: # the new observations are already too far apart if not shard_station: values.update({ 'created': created, 'block_first': self.today, 'block_last': self.today, 'block_count': 1, }) return ('new_moving', values) else: block_count = shard_station.block_count or 0 values.update({ 'lat': None, 'lon': None, 'max_lat': None, 'min_lat': None, 'max_lon': None, 'min_lon': None, 'country': shard_station.country, 'radius': None, 'samples': None, 'source': None, 'block_last': self.today, 'block_count': block_count + 1, }) return ('moving', values) if shard_station is None: # totally new station, only agreeing observations radius = circle_radius( obs_new_lat, obs_new_lon, obs_max_lat, obs_max_lon, obs_min_lat, obs_min_lon) values.update({ 'created': created, 'lat': obs_new_lat, 'lon': obs_new_lon, 'max_lat': float(obs_max_lat), 'min_lat': float(obs_min_lat), 'max_lon': float(obs_max_lon), 'min_lon': float(obs_min_lon), 'country': country_for_location(obs_new_lat, obs_new_lon), 'radius': radius, 'samples': obs_length, 'source': None, }) return ('new', values) else: # shard_station + new observations positions = numpy.append(obs_positions, [ (numpy.nan if shard_station.lat is None else shard_station.lat, numpy.nan if shard_station.lon is None else shard_station.lon), (numpy.nan if shard_station.max_lat is None else shard_station.max_lat, numpy.nan if shard_station.max_lon is None else shard_station.max_lon), (numpy.nan if shard_station.min_lat is None else shard_station.min_lat, numpy.nan if shard_station.min_lon is None else shard_station.min_lon), ], axis=0) max_lat, max_lon = numpy.nanmax(positions, axis=0) min_lat, min_lon = numpy.nanmin(positions, axis=0) box_dist = distance(min_lat, min_lon, max_lat, max_lon) if box_dist > self.max_dist_meters: # shard_station + disagreeing observations block_count = shard_station.block_count or 0 values.update({ 'lat': None, 'lon': None, 'max_lat': None, 'min_lat': None, 'max_lon': None, 'min_lon': None, 'country': shard_station.country, 'radius': None, 'samples': None, 'source': None, 'block_last': self.today, 'block_count': block_count + 1, }) return ('moving', values) else: # shard_station + agreeing observations if shard_station.lat is None or shard_station.lon is None: old_weight = 0 else: old_weight = min((shard_station.samples or 0), self.MAX_OLD_OBSERVATIONS) new_lat = ((obs_new_lat * obs_length + (shard_station.lat or 0.0) * old_weight) / (obs_length + old_weight)) new_lon = ((obs_new_lon * obs_length + (shard_station.lon or 0.0) * old_weight) / (obs_length + old_weight)) samples = (shard_station.samples or 0) + obs_length radius = circle_radius( new_lat, new_lon, max_lat, max_lon, min_lat, min_lon) country = shard_station.country if (country and not country_matches_location( new_lat, new_lon, country)): # reset country if it no longer matches country = None if not country: country = country_for_location(new_lat, new_lon) values.update({ 'lat': new_lat, 'lon': new_lon, 'max_lat': float(max_lat), 'min_lat': float(min_lat), 'max_lon': float(max_lon), 'min_lon': float(min_lon), 'country': country, 'radius': radius, 'samples': samples, 'source': None, # use the exact same keys as in the moving case 'block_last': shard_station.block_last, 'block_count': shard_station.block_count, }) return ('changed', values) return (None, None) # pragma: no cover
def new_station_values(self, station, station_key, first_blocked, observations): # This function returns a 3-tuple, the first element is True, # if the station was found to be moving. # The second element is either None or a dict of values, # if the station is new and should result in a table insert # The third element is either None or a dict of values # if the station did exist and should be updated obs_length = len(observations) obs_positions = numpy.array( [(obs.lat, obs.lon) for obs in observations], dtype=numpy.double) obs_lat, obs_lon = centroid(obs_positions) values = { 'modified': self.utcnow, } values.update(station_key.__dict__) if self.station_type == 'cell': # pass on extra psc column which is not actually part # of the stations hash key values['psc'] = observations[-1].psc created = self.utcnow if station is None: if first_blocked: # if the station did previously exist, retain at least the # time it was first put on a blocklist as the creation date created = first_blocked values.update({ 'created': created, 'range': 0, 'total_measures': 0, }) if (station is not None and station.lat is not None and station.lon is not None): obs_positions = numpy.append(obs_positions, [ (station.lat, station.lon), (numpy.nan if station.max_lat is None else station.max_lat, numpy.nan if station.max_lon is None else station.max_lon), (numpy.nan if station.min_lat is None else station.min_lat, numpy.nan if station.min_lon is None else station.min_lon), ], axis=0) existing_station = True else: values['lat'] = obs_lat values['lon'] = obs_lon existing_station = False max_lat, max_lon = numpy.nanmax(obs_positions, axis=0) min_lat, min_lon = numpy.nanmin(obs_positions, axis=0) # calculate sphere-distance from opposite corners of # bounding box containing current location estimate # and new observations; if too big, station is moving box_dist = distance(min_lat, min_lon, max_lat, max_lon) # TODO: If we get a too large box_dist, we should not create # a new station record with the impossibly big distance, # so moving the box_dist > self.max_dist_meters here if existing_station: if box_dist > self.max_dist_meters: # Signal a moving station and return early without updating # the station since it will be deleted by caller momentarily return (True, None, None) # limit the maximum weight of the old station estimate old_weight = min(station.total_measures, self.MAX_OLD_OBSERVATIONS) new_weight = old_weight + obs_length values['lat'] = ((station.lat * old_weight) + (obs_lat * obs_length)) / new_weight values['lon'] = ((station.lon * old_weight) + (obs_lon * obs_length)) / new_weight # increase total counter if station is not None: values['total_measures'] = station.total_measures + obs_length else: values['total_measures'] = obs_length # update max/min lat/lon columns values['min_lat'] = float(min_lat) values['min_lon'] = float(min_lon) values['max_lat'] = float(max_lat) values['max_lon'] = float(max_lon) # give radio-range estimate between extreme values and centroid values['range'] = circle_radius( values['lat'], values['lon'], max_lat, max_lon, min_lat, min_lon) if station is None: # return new values return (False, values, None) else: # return updated values, remove station from session self.session.expunge(station) return (False, None, values)
def update_area(self, area_key): # Select all cells in this area and derive a bounding box for them cell_query = (self.cell_model.querykey(self.session, area_key) .filter(self.cell_model.lat.isnot(None)) .filter(self.cell_model.lon.isnot(None))) cells = cell_query.all() area_query = self.area_model.querykey(self.session, area_key) if len(cells) == 0: # If there are no more underlying cells, delete the area entry area_query.delete() else: # Otherwise update the area entry based on all the cells area = area_query.first() cell_extremes = numpy.array([ (numpy.nan if cell.max_lat is None else cell.max_lat, numpy.nan if cell.max_lon is None else cell.max_lon) for cell in cells] + [ (numpy.nan if cell.min_lat is None else cell.min_lat, numpy.nan if cell.min_lon is None else cell.min_lon) for cell in cells ], dtype=numpy.double) max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0) min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0) ctr_lat, ctr_lon = centroid( numpy.array([(c.lat, c.lon) for c in cells], dtype=numpy.double)) radius = circle_radius( ctr_lat, ctr_lon, max_lat, max_lon, min_lat, min_lon) # Now create or update the area cell_ranges = numpy.array([ (numpy.nan if cell.range is None else cell.range) for cell in cells ], dtype=numpy.int32) avg_cell_range = int(round(numpy.nanmean(cell_ranges))) num_cells = len(cells) if area is None: stmt = self.area_model.__table__.insert( mysql_on_duplicate='num_cells = num_cells' # no-op ).values( created=self.utcnow, modified=self.utcnow, lat=ctr_lat, lon=ctr_lon, range=radius, avg_cell_range=avg_cell_range, num_cells=num_cells, **area_key.__dict__ ) self.session.execute(stmt) else: area.modified = self.utcnow area.lat = ctr_lat area.lon = ctr_lon area.range = radius area.avg_cell_range = avg_cell_range area.num_cells = num_cells
def station_values(self, station_key, shard_station, observations): """ Return two-tuple of status, value dict where status is one of: `new`, `new_moving`, `moving`, `changed`. """ # cases: # we always get a station key and observations # 0. observations disagree # 0.a. no shard station, return new_moving # 0.b. shard station, return moving # 1. no shard station # 1.a. obs agree -> return new # 2. shard station # 2.a. obs disagree -> return moving # 2.b. obs agree -> return changed created = self.utcnow values = self._base_station_values(station_key, observations) obs_positions = numpy.array( [(obs.lat, obs.lon) for obs in observations], dtype=numpy.double) obs_length = len(observations) obs_weights = numpy.array( [obs.weight for obs in observations], dtype=numpy.double) obs_weight = float(obs_weights.sum()) obs_new_lat, obs_new_lon = numpy.average( obs_positions, axis=0, weights=obs_weights) obs_new_lat = float(obs_new_lat) obs_new_lon = float(obs_new_lon) obs_max_lat, obs_max_lon = obs_positions.max(axis=0) obs_min_lat, obs_min_lon = obs_positions.min(axis=0) obs_box_dist = distance(obs_min_lat, obs_min_lon, obs_max_lat, obs_max_lon) if obs_box_dist > self.max_dist_meters: # the new observations are already too far apart if not shard_station: values.update({ 'created': created, 'block_first': self.today, 'block_last': self.today, 'block_count': 1, }) return ('new_moving', values) else: block_count = shard_station.block_count or 0 values.update({ 'lat': None, 'lon': None, 'max_lat': None, 'min_lat': None, 'max_lon': None, 'min_lon': None, 'radius': None, 'region': shard_station.region, 'samples': None, 'source': None, 'weight': None, 'block_first': shard_station.block_first or self.today, 'block_last': self.today, 'block_count': block_count + 1, }) return ('moving', values) if shard_station is None: # totally new station, only agreeing observations radius = circle_radius( obs_new_lat, obs_new_lon, obs_max_lat, obs_max_lon, obs_min_lat, obs_min_lon) values.update({ 'created': created, 'lat': obs_new_lat, 'lon': obs_new_lon, 'max_lat': float(obs_max_lat), 'min_lat': float(obs_min_lat), 'max_lon': float(obs_max_lon), 'min_lon': float(obs_min_lon), 'radius': radius, 'region': GEOCODER.region(obs_new_lat, obs_new_lon), 'samples': obs_length, 'source': None, 'weight': obs_weight, }) return ('new', values) else: # shard_station + new observations positions = numpy.append(obs_positions, [ (numpy.nan if shard_station.lat is None else shard_station.lat, numpy.nan if shard_station.lon is None else shard_station.lon), (numpy.nan if shard_station.max_lat is None else shard_station.max_lat, numpy.nan if shard_station.max_lon is None else shard_station.max_lon), (numpy.nan if shard_station.min_lat is None else shard_station.min_lat, numpy.nan if shard_station.min_lon is None else shard_station.min_lon), ], axis=0) max_lat, max_lon = numpy.nanmax(positions, axis=0) min_lat, min_lon = numpy.nanmin(positions, axis=0) box_dist = distance(min_lat, min_lon, max_lat, max_lon) if box_dist > self.max_dist_meters: # shard_station + disagreeing observations block_count = shard_station.block_count or 0 values.update({ 'lat': None, 'lon': None, 'max_lat': None, 'min_lat': None, 'max_lon': None, 'min_lon': None, 'radius': None, 'region': shard_station.region, 'samples': None, 'source': None, 'weight': None, 'block_first': shard_station.block_first or self.today, 'block_last': self.today, 'block_count': block_count + 1, }) return ('moving', values) else: # shard_station + agreeing observations if shard_station.lat is None or shard_station.lon is None: old_weight = 0 else: old_weight = min((shard_station.weight or 0.0), self.MAX_OLD_WEIGHT) new_lat = ((obs_new_lat * obs_weight + (shard_station.lat or 0.0) * old_weight) / (obs_weight + old_weight)) new_lon = ((obs_new_lon * obs_weight + (shard_station.lon or 0.0) * old_weight) / (obs_weight + old_weight)) # put in maximum value to avoid overflow of DB column samples = min((shard_station.samples or 0) + obs_length, 4294967295) weight = min((shard_station.weight or 0.0) + obs_weight, 1000000000.0) radius = circle_radius( new_lat, new_lon, max_lat, max_lon, min_lat, min_lon) region = shard_station.region if (region and not GEOCODER.in_region( new_lat, new_lon, region)): # reset region if it no longer matches region = None if not region: region = GEOCODER.region(new_lat, new_lon) values.update({ 'lat': new_lat, 'lon': new_lon, 'max_lat': float(max_lat), 'min_lat': float(min_lat), 'max_lon': float(max_lon), 'min_lon': float(min_lon), 'radius': radius, 'region': region, 'samples': samples, 'source': None, 'weight': weight, # use the exact same keys as in the moving case 'block_first': shard_station.block_first, 'block_last': shard_station.block_last, 'block_count': shard_station.block_count, }) return ('changed', values) return (None, None) # pragma: no cover
def update_area(self, session, areaid): # Select all cells in this area and derive a bounding box for them radio, mcc, mnc, lac = decode_cellarea(areaid) load_fields = ('cellid', 'lat', 'lon', 'radius', 'region', 'last_seen', 'max_lat', 'max_lon', 'min_lat', 'min_lon') shard = self.cell_model.shard_model(radio) fields = [getattr(shard.__table__.c, f) for f in load_fields] cells = session.execute( select(fields) .where(shard.__table__.c.radio == radio) .where(shard.__table__.c.mcc == mcc) .where(shard.__table__.c.mnc == mnc) .where(shard.__table__.c.lac == lac) .where(shard.__table__.c.lat.isnot(None)) .where(shard.__table__.c.lon.isnot(None)) ).fetchall() if len(cells) == 0: # If there are no more underlying cells, delete the area entry session.execute( delete(self.area_table) .where(self.area_table.c.areaid == areaid) ) return # Otherwise update the area entry based on all the cells area = session.execute( select([self.area_table.c.areaid, self.area_table.c.modified, self.area_table.c.lat, self.area_table.c.lon, self.area_table.c.radius, self.area_table.c.region, self.area_table.c.avg_cell_radius, self.area_table.c.num_cells, self.area_table.c.last_seen, ]) .where(self.area_table.c.areaid == areaid) ).fetchone() cell_extremes = numpy.array([ (numpy.nan if cell.max_lat is None else cell.max_lat, numpy.nan if cell.max_lon is None else cell.max_lon) for cell in cells] + [ (numpy.nan if cell.min_lat is None else cell.min_lat, numpy.nan if cell.min_lon is None else cell.min_lon) for cell in cells ], dtype=numpy.double) max_lat, max_lon = numpy.nanmax(cell_extremes, axis=0) min_lat, min_lon = numpy.nanmin(cell_extremes, axis=0) ctr_lat, ctr_lon = numpy.array( [(c.lat, c.lon) for c in cells], dtype=numpy.double).mean(axis=0) ctr_lat = float(ctr_lat) ctr_lon = float(ctr_lon) radius = circle_radius( ctr_lat, ctr_lon, max_lat, max_lon, min_lat, min_lon) cell_radii = numpy.array([ (numpy.nan if cell.radius is None else cell.radius) for cell in cells ], dtype=numpy.int32) avg_cell_radius = int(round(numpy.nanmean(cell_radii))) num_cells = len(cells) region = self.region(ctr_lat, ctr_lon, mcc, cells) last_seen = None cell_last_seen = set([cell.last_seen for cell in cells if cell.last_seen is not None]) if cell_last_seen: last_seen = max(cell_last_seen) if area is None: session.execute( self.area_table.insert( mysql_on_duplicate='num_cells = num_cells' # no-op ).values( areaid=areaid, radio=radio, mcc=mcc, mnc=mnc, lac=lac, created=self.utcnow, modified=self.utcnow, lat=ctr_lat, lon=ctr_lon, radius=radius, region=region, avg_cell_radius=avg_cell_radius, num_cells=num_cells, last_seen=last_seen, ) ) else: session.execute( self.area_table.update() .where(self.area_table.c.areaid == areaid) .values( modified=self.utcnow, lat=ctr_lat, lon=ctr_lon, radius=radius, region=region, avg_cell_radius=avg_cell_radius, num_cells=num_cells, last_seen=last_seen, ) )