def aggregate_station_obs(self):
station = self.station
obs_data = self.obs_data
def get_nan(name):
value = getattr(station, name, None)
return numpy.nan if value is None else value
positions = numpy.append(
obs_data["positions"],
[
(get_nan("lat"), get_nan("lon")),
(get_nan("max_lat"), get_nan("max_lon")),
(get_nan("min_lat"), get_nan("min_lon")),
],
axis=0,
)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
if station.lat is None or station.lon is None:
old_weight = 0.0
else:
old_weight = min((station.weight or 0.0), self.MAX_OLD_WEIGHT)
lat = (
obs_data["lat"] * obs_data["weight"] + (station.lat or 0.0) * old_weight
) / (obs_data["weight"] + old_weight)
lon = (
obs_data["lon"] * obs_data["weight"] + (station.lon or 0.0) * old_weight
) / (obs_data["weight"] + old_weight)
radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon)
region = station.region
if region and not GEOCODER.in_region(lat, lon, region):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(lat, lon)
samples, weight = self.bounded_samples_weight(
(station.samples or 0) + obs_data["samples"],
(station.weight or 0.0) + obs_data["weight"],
)
return {
"lat": lat,
"lon": lon,
"max_lat": float(max_lat),
"min_lat": float(min_lat),
"max_lon": float(max_lon),
"min_lon": float(min_lon),
"radius": radius,
"region": region,
"samples": samples,
"weight": weight,
}
def aggregate_station_obs(self):
station = self.station
obs_data = self.obs_data
def get_nan(name):
value = getattr(station, name, None)
return numpy.nan if value is None else value
positions = numpy.append(obs_data['positions'], [
(get_nan('lat'), get_nan('lon')),
(get_nan('max_lat'), get_nan('max_lon')),
(get_nan('min_lat'), get_nan('min_lon')),
],
axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
if station.lat is None or station.lon is None:
old_weight = 0.0
else:
old_weight = min((station.weight or 0.0), self.MAX_OLD_WEIGHT)
lat = ((obs_data['lat'] * obs_data['weight'] +
(station.lat or 0.0) * old_weight) /
(obs_data['weight'] + old_weight))
lon = ((obs_data['lon'] * obs_data['weight'] +
(station.lon or 0.0) * old_weight) /
(obs_data['weight'] + old_weight))
radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon)
region = station.region
if (region and not GEOCODER.in_region(lat, lon, region)):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(lat, lon)
samples, weight = self.bounded_samples_weight(
(station.samples or 0) + obs_data['samples'],
(station.weight or 0.0) + obs_data['weight'])
return {
'lat': lat,
'lon': lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'radius': radius,
'region': region,
'samples': samples,
'weight': weight,
}
def aggregate_station_obs(self):
station = self.station
obs_data = self.obs_data
def get_nan(name):
value = getattr(station, name, None)
return numpy.nan if value is None else value
positions = numpy.append(obs_data['positions'], [
(get_nan('lat'), get_nan('lon')),
(get_nan('max_lat'), get_nan('max_lon')),
(get_nan('min_lat'), get_nan('min_lon')),
], axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
if station.lat is None or station.lon is None:
old_weight = 0.0
else:
old_weight = min((station.weight or 0.0), self.MAX_OLD_WEIGHT)
lat = ((obs_data['lat'] * obs_data['weight'] +
(station.lat or 0.0) * old_weight) /
(obs_data['weight'] + old_weight))
lon = ((obs_data['lon'] * obs_data['weight'] +
(station.lon or 0.0) * old_weight) /
(obs_data['weight'] + old_weight))
radius = circle_radius(lat, lon, max_lat, max_lon, min_lat, min_lon)
region = station.region
if (region and not GEOCODER.in_region(lat, lon, region)):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(lat, lon)
samples, weight = self.bounded_samples_weight(
(station.samples or 0) + obs_data['samples'],
(station.weight or 0.0) + obs_data['weight'])
return {
'lat': lat, 'lon': lon,
'max_lat': float(max_lat), 'min_lat': float(min_lat),
'max_lon': float(max_lon), 'min_lon': float(min_lon),
'radius': radius, 'region': region,
'samples': samples, 'weight': weight,
}
def station_values(self, station_key, shard_station, observations):
"""
Return two-tuple of status, value dict where status is one of:
`new`, `new_moving`, `moving`, `changed`.
"""
# cases:
# we always get a station key and observations
# 0. observations disagree
# 0.a. no shard station, return new_moving
# 0.b. shard station, return moving
# 1. no shard station
# 1.a. obs agree -> return new
# 2. shard station
# 2.a. obs disagree -> return moving
# 2.b. obs agree -> return changed
created = self.utcnow
values = self._base_station_values(station_key, observations)
obs_length = len(observations)
obs_positions = numpy.array([(obs.lat, obs.lon)
for obs in observations],
dtype=numpy.double)
obs_new_lat, obs_new_lon = centroid(obs_positions)
obs_max_lat, obs_max_lon = numpy.nanmax(obs_positions, axis=0)
obs_min_lat, obs_min_lon = numpy.nanmin(obs_positions, axis=0)
obs_box_dist = distance(obs_min_lat, obs_min_lon, obs_max_lat,
obs_max_lon)
if obs_box_dist > self.max_dist_meters:
# the new observations are already too far apart
if not shard_station:
values.update({
'created': created,
'block_first': self.today,
'block_last': self.today,
'block_count': 1,
})
return ('new_moving', values)
else:
block_count = shard_station.block_count or 0
values.update({
'lat':
None,
'lon':
None,
'max_lat':
None,
'min_lat':
None,
'max_lon':
None,
'min_lon':
None,
'radius':
None,
'region':
shard_station.region,
'samples':
None,
'source':
None,
'block_first':
shard_station.block_first or self.today,
'block_last':
self.today,
'block_count':
block_count + 1,
})
return ('moving', values)
if shard_station is None:
# totally new station, only agreeing observations
radius = circle_radius(obs_new_lat, obs_new_lon, obs_max_lat,
obs_max_lon, obs_min_lat, obs_min_lon)
values.update({
'created': created,
'lat': obs_new_lat,
'lon': obs_new_lon,
'max_lat': float(obs_max_lat),
'min_lat': float(obs_min_lat),
'max_lon': float(obs_max_lon),
'min_lon': float(obs_min_lon),
'radius': radius,
'region': GEOCODER.region(obs_new_lat, obs_new_lon),
'samples': obs_length,
'source': None,
})
return ('new', values)
else:
# shard_station + new observations
positions = numpy.append(obs_positions, [
(numpy.nan if shard_station.lat is None else shard_station.lat,
numpy.nan
if shard_station.lon is None else shard_station.lon),
(numpy.nan if shard_station.max_lat is None else
shard_station.max_lat, numpy.nan
if shard_station.max_lon is None else shard_station.max_lon),
(numpy.nan if shard_station.min_lat is None else
shard_station.min_lat, numpy.nan
if shard_station.min_lon is None else shard_station.min_lon),
],
axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
box_dist = distance(min_lat, min_lon, max_lat, max_lon)
if box_dist > self.max_dist_meters:
# shard_station + disagreeing observations
block_count = shard_station.block_count or 0
values.update({
'lat':
None,
'lon':
None,
'max_lat':
None,
'min_lat':
None,
'max_lon':
None,
'min_lon':
None,
'radius':
None,
'region':
shard_station.region,
'samples':
None,
'source':
None,
'block_first':
shard_station.block_first or self.today,
'block_last':
self.today,
'block_count':
block_count + 1,
})
return ('moving', values)
else:
# shard_station + agreeing observations
if shard_station.lat is None or shard_station.lon is None:
old_weight = 0
else:
old_weight = min((shard_station.samples or 0),
self.MAX_OLD_OBSERVATIONS)
new_lat = ((obs_new_lat * obs_length +
(shard_station.lat or 0.0) * old_weight) /
(obs_length + old_weight))
new_lon = ((obs_new_lon * obs_length +
(shard_station.lon or 0.0) * old_weight) /
(obs_length + old_weight))
samples = (shard_station.samples or 0) + obs_length
radius = circle_radius(new_lat, new_lon, max_lat, max_lon,
min_lat, min_lon)
region = shard_station.region
if (region
and not GEOCODER.in_region(new_lat, new_lon, region)):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(new_lat, new_lon)
values.update({
'lat': new_lat,
'lon': new_lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'radius': radius,
'region': region,
'samples': samples,
'source': None,
# use the exact same keys as in the moving case
'block_first': shard_station.block_first,
'block_last': shard_station.block_last,
'block_count': shard_station.block_count,
})
return ('changed', values)
return (None, None) # pragma: no cover
def station_values(self, station_key, shard_station, observations):
"""
Return two-tuple of status, value dict where status is one of:
`new`, `new_moving`, `moving`, `changed`.
"""
# cases:
# we always get a station key and observations
# 0. observations disagree
# 0.a. no shard station, return new_moving
# 0.b. shard station, return moving
# 1. no shard station
# 1.a. obs agree -> return new
# 2. shard station
# 2.a. obs disagree -> return moving
# 2.b. obs agree -> return changed
created = self.utcnow
values = self._base_station_values(station_key, observations)
obs_positions = numpy.array(
[(obs.lat, obs.lon) for obs in observations],
dtype=numpy.double)
obs_length = len(observations)
obs_weights = numpy.array(
[obs.weight for obs in observations],
dtype=numpy.double)
obs_weight = float(obs_weights.sum())
obs_new_lat, obs_new_lon = numpy.average(
obs_positions, axis=0, weights=obs_weights)
obs_new_lat = float(obs_new_lat)
obs_new_lon = float(obs_new_lon)
obs_max_lat, obs_max_lon = obs_positions.max(axis=0)
obs_min_lat, obs_min_lon = obs_positions.min(axis=0)
obs_box_dist = distance(obs_min_lat, obs_min_lon,
obs_max_lat, obs_max_lon)
if obs_box_dist > self.max_dist_meters:
# the new observations are already too far apart
if not shard_station:
values.update({
'created': created,
'block_first': self.today,
'block_last': self.today,
'block_count': 1,
})
return ('new_moving', values)
else:
block_count = shard_station.block_count or 0
values.update({
'lat': None,
'lon': None,
'max_lat': None,
'min_lat': None,
'max_lon': None,
'min_lon': None,
'radius': None,
'region': shard_station.region,
'samples': None,
'source': None,
'weight': None,
'block_first': shard_station.block_first or self.today,
'block_last': self.today,
'block_count': block_count + 1,
})
return ('moving', values)
if shard_station is None:
# totally new station, only agreeing observations
radius = circle_radius(
obs_new_lat, obs_new_lon,
obs_max_lat, obs_max_lon, obs_min_lat, obs_min_lon)
values.update({
'created': created,
'lat': obs_new_lat,
'lon': obs_new_lon,
'max_lat': float(obs_max_lat),
'min_lat': float(obs_min_lat),
'max_lon': float(obs_max_lon),
'min_lon': float(obs_min_lon),
'radius': radius,
'region': GEOCODER.region(obs_new_lat, obs_new_lon),
'samples': obs_length,
'source': None,
'weight': obs_weight,
})
return ('new', values)
else:
# shard_station + new observations
positions = numpy.append(obs_positions, [
(numpy.nan if shard_station.lat is None
else shard_station.lat,
numpy.nan if shard_station.lon is None
else shard_station.lon),
(numpy.nan if shard_station.max_lat is None
else shard_station.max_lat,
numpy.nan if shard_station.max_lon is None
else shard_station.max_lon),
(numpy.nan if shard_station.min_lat is None
else shard_station.min_lat,
numpy.nan if shard_station.min_lon is None
else shard_station.min_lon),
], axis=0)
max_lat, max_lon = numpy.nanmax(positions, axis=0)
min_lat, min_lon = numpy.nanmin(positions, axis=0)
box_dist = distance(min_lat, min_lon, max_lat, max_lon)
if box_dist > self.max_dist_meters:
# shard_station + disagreeing observations
block_count = shard_station.block_count or 0
values.update({
'lat': None,
'lon': None,
'max_lat': None,
'min_lat': None,
'max_lon': None,
'min_lon': None,
'radius': None,
'region': shard_station.region,
'samples': None,
'source': None,
'weight': None,
'block_first': shard_station.block_first or self.today,
'block_last': self.today,
'block_count': block_count + 1,
})
return ('moving', values)
else:
# shard_station + agreeing observations
if shard_station.lat is None or shard_station.lon is None:
old_weight = 0
else:
old_weight = min((shard_station.weight or 0.0),
self.MAX_OLD_WEIGHT)
new_lat = ((obs_new_lat * obs_weight +
(shard_station.lat or 0.0) * old_weight) /
(obs_weight + old_weight))
new_lon = ((obs_new_lon * obs_weight +
(shard_station.lon or 0.0) * old_weight) /
(obs_weight + old_weight))
# put in maximum value to avoid overflow of DB column
samples = min((shard_station.samples or 0) + obs_length,
4294967295)
weight = min((shard_station.weight or 0.0) + obs_weight,
1000000000.0)
radius = circle_radius(
new_lat, new_lon, max_lat, max_lon, min_lat, min_lon)
region = shard_station.region
if (region and not GEOCODER.in_region(
new_lat, new_lon, region)):
# reset region if it no longer matches
region = None
if not region:
region = GEOCODER.region(new_lat, new_lon)
values.update({
'lat': new_lat,
'lon': new_lon,
'max_lat': float(max_lat),
'min_lat': float(min_lat),
'max_lon': float(max_lon),
'min_lon': float(min_lon),
'radius': radius,
'region': region,
'samples': samples,
'source': None,
'weight': weight,
# use the exact same keys as in the moving case
'block_first': shard_station.block_first,
'block_last': shard_station.block_last,
'block_count': shard_station.block_count,
})
return ('changed', values)
return (None, None) # pragma: no cover