def _is_valid_reading(sensor_data: typing.Dict[str, typing.Any]) -> bool:
if sensor_data["last_seen"] < timestamp() - (60 * 60):
# Out of date / maybe dead
return False
if sensor_data["channel_flags"] != "Normal":
# Flagged for an unusually high reading
return False
try:
pm25 = float(sensor_data["pm2.5"])
except (TypeError, ValueError):
return False
if math.isnan(pm25):
# Purpleair can occasionally return NaN.
# I wonder if this is a bug on their end.
return False
if pm25 <= 0 or pm25 > 1000:
# Something is very wrong
return False
try:
humidity = float(sensor_data["humidity"])
except (TypeError, ValueError):
return False
if math.isnan(humidity):
return False
latitude = sensor_data["latitude"]
longitude = sensor_data["longitude"]
if latitude is None or longitude is None:
return False
return True
def maybe_notify(self) -> bool:
if not self.is_in_send_window:
return False
if self.last_alert_sent_at >= timestamp() - self.FREQUENCY:
return False
curr_pm25 = self.zipcode.pm25
curr_aqi_level = Pm25.from_measurement(curr_pm25)
curr_aqi = pm25_to_aqi(curr_pm25)
# Only send if the pm25 changed a level since the last time we sent this alert.
last_aqi_level = Pm25.from_measurement(self.last_pm25)
if curr_aqi_level == last_aqi_level:
return False
# Do not alert clients who received an alert recently unless AQI has changed markedly.
was_alerted_recently = self.last_alert_sent_at > timestamp() - (60 *
60 * 6)
last_aqi = self.last_aqi
if (was_alerted_recently and last_aqi and curr_aqi
and abs(curr_aqi - last_aqi) < 20):
return False
message = (
"Air quality in {city} {zipcode} has changed to {curr_aqi_level} (AQI {curr_aqi}).\n"
"\n"
'Reply "M" for Menu or "E" to end alerts.').format(
city=self.zipcode.city.name,
zipcode=self.zipcode.zipcode,
curr_aqi_level=curr_aqi_level.display,
curr_aqi=curr_aqi,
)
if not self.send_message(message):
return False
self.last_alert_sent_at = timestamp()
self.last_pm25 = curr_pm25
self.num_alerts_sent += 1
db.session.commit()
self.log_event(EventType.ALERT,
zipcode=self.zipcode.zipcode,
pm25=curr_pm25)
return True
def disable_alerts(self, is_automatic=False):
if self.alerts_disabled_at == 0:
self.alerts_disabled_at = timestamp()
db.session.commit()
self.log_event(
EventType.UNSUBSCRIBE_AUTO
if is_automatic else EventType.UNSUBSCRIBE,
zipcode=self.zipcode.zipcode,
)
def get_activity_counts(self):
windows = [1, 2, 3, 4, 5, 6, 7, 30]
curr_time = timestamp()
counts = {window: 0 for window in windows}
for client in self.filter_phones().all():
for window in windows:
ts = curr_time - (window * 24 * 60 * 60)
if client.last_activity_at > ts or client.last_alert_sent_at > ts:
counts[window] += 1
return counts
def admin_bulk_sms():
if not current_user.can_send_sms:
return redirect(url_for("admin_summary"))
form = BulkSMSForm(last_active_at=now())
if form.validate_on_submit():
bulk_send.delay(form.data["message"],
form.data["last_active_at"].timestamp())
flash("Sent!")
return redirect(url_for("admin_summary"))
return render_template(
"bulk_sms.html",
form=form,
num_inactive=Client.query.filter_inactive_since(timestamp()).count(),
)
def get_or_create(
self, identifier: str,
type_code: ClientIdentifierType) -> typing.Tuple["Client", bool]:
client = self.filter_by(identifier=identifier,
type_code=type_code).first()
if not client:
client = Client(identifier=identifier,
type_code=type_code,
last_activity_at=timestamp())
db.session.add(client)
db.session.commit()
was_created = True
else:
was_created = False
return client, was_created
def _metrics_sync():
updates = []
ts = timestamp()
zipcodes_to_sensors = collections.defaultdict(list)
for zipcode_id, latest_reading, distance in (
Sensor.query.join(SensorZipcodeRelation).filter(
Sensor.updated_at > ts - (30 * 60)).with_entities(
SensorZipcodeRelation.zipcode_id,
Sensor.latest_reading,
SensorZipcodeRelation.distance,
).all()):
zipcodes_to_sensors[zipcode_id].append((latest_reading, distance))
for zipcode_id, sensor_tuples in zipcodes_to_sensors.items():
readings: typing.List[float] = []
closest_reading = float("inf")
farthest_reading = 0.0
for reading, distance in sorted(sensor_tuples, key=lambda s: s[1]):
if (len(readings) < DESIRED_NUM_READINGS
or distance < DESIRED_READING_DISTANCE_KM):
readings.append(reading)
closest_reading = min(distance, closest_reading)
farthest_reading = max(distance, farthest_reading)
else:
break
if readings:
pm25 = round(sum(readings) / len(readings), ndigits=3)
num_sensors = len(readings)
min_sensor_distance = round(closest_reading, ndigits=3)
max_sensor_distance = round(farthest_reading, ndigits=3)
updates.append({
"id": zipcode_id,
"pm25": pm25,
"pm25_updated_at": ts,
"num_sensors": num_sensors,
"min_sensor_distance": min_sensor_distance,
"max_sensor_distance": max_sensor_distance,
})
logger.info("Updating %s zipcodes", len(updates))
for mappings in chunk_list(updates, batch_size=5000):
db.session.bulk_update_mappings(Zipcode, mappings)
db.session.commit()
def log_request(self, zipcode: Zipcode):
request = Request.query.filter_by(
client_id=self.id,
zipcode_id=zipcode.id,
).first()
now = timestamp()
if request is None:
request = Request(
client_id=self.id,
zipcode_id=zipcode.id,
count=1,
first_ts=now,
last_ts=now,
)
db.session.add(request)
else:
request.count += 1
request.last_ts = now
db.session.commit()
def admin_bulk_sms():
if not current_user.can_send_sms:
return redirect(url_for("admin_summary"))
form = BulkSMSForm(last_active_at=now(), include_unsubscribed=False)
if form.validate_on_submit():
bulk_send.delay(
message=form.data["message"],
last_active_at=form.data["last_active_at"].timestamp(),
locale=form.data["locale"],
include_unsubscribed=form.data["include_unsubscribed"],
is_feedback_request=form.data["is_feedback_request"],
)
flash("Sent!")
return redirect(url_for("admin_summary"))
return render_template(
"bulk_sms.html",
form=form,
num_inactive=Client.query.filter_inactive_since(
timestamp(), form.data["include_unsubscribed"]).count(),
)
def _get_purpleair_sensors_data() -> typing.List[typing.Dict[str, typing.Any]]:
logger = get_celery_logger()
try:
response_dict = call_purpleair_sensors_api().json()
except (requests.RequestException, json.JSONDecodeError) as e:
# Send an email to an admin if data lags by more than 30 minutes.
# Otherwise, just log a warning as most of these errors are
# transient. In the future we might choose to retry on transient
# failures, but it's not urgent since we will rerun the sync
# every ten minutes anyway.
last_updated_at = Sensor.query.get_last_updated_at()
seconds_since_last_update = timestamp() - last_updated_at
if seconds_since_last_update > 30 * 60:
level = logging.ERROR
else:
level = logging.WARNING
logger.log(
level,
"%s updating purpleair data: %s",
type(e).__name__,
e,
exc_info=True,
)
return []
else:
fields = response_dict["fields"]
channel_flags = response_dict["channel_flags"]
data = []
for sensor_data in response_dict["data"]:
sensor_data = dict(zip(fields, sensor_data))
try:
sensor_data["channel_flags"] = channel_flags[
sensor_data["channel_flags"]]
except KeyError:
pass
data.append(sensor_data)
return data
def _is_valid_reading(sensor_data: typing.Dict[str, typing.Any]) -> bool:
if sensor_data.get("DEVICE_LOCATIONTYPE") != "outside":
return False
if sensor_data.get("ParentID"):
return False
if sensor_data.get("LastSeen", 0) < timestamp() - (60 * 60):
# Out of date / maybe dead
return False
if sensor_data.get("Flag"):
# Flagged for an unusually high reading
return False
try:
pm25 = float(sensor_data.get("PM2_5Value", 0))
except (TypeError, ValueError):
return False
if pm25 <= 0 or pm25 > 1000:
# Something is very wrong
return False
latitude = sensor_data.get("Lat")
longitude = sensor_data.get("Lon")
if latitude is None or longitude is None:
return False
return True
def mark_seen(self, locale: str):
self.last_activity_at = timestamp()
if self.locale != locale:
self.locale = locale
db.session.commit()
def get_share_window(self) -> typing.Tuple[int, int]:
ts = timestamp()
share_window_start = ts - 60 * 15
share_window_end = ts - 60 * 5
return share_window_start, share_window_end
def filter_eligible_for_sending(self) -> "ClientQuery":
cutoff = timestamp() - Client.FREQUENCY
return (self.filter_phones().options(joinedload(
Client.zipcode)).filter(Client.alerts_disabled_at == 0).filter(
Client.last_alert_sent_at < cutoff).filter(
Client.zipcode_id.isnot(None)))
def mark_seen(self):
self.last_activity_at = timestamp()
db.session.commit()
def maybe_notify(self) -> bool:
if not self.is_in_send_window:
return False
alert_frequency = self.alert_frequency * 60 * 60
if self.last_alert_sent_at >= timestamp() - alert_frequency:
return False
curr_pm25 = self.get_current_pm25()
curr_aqi_level = self.get_current_pm25_level()
curr_aqi = self.get_current_aqi()
# Only send if the pm25 changed a level since the last time we sent this alert.
last_aqi_level = self.get_last_pm25_level()
if curr_aqi_level == last_aqi_level:
return False
alert_threshold = self.alert_threshold
# If the current AQI is below the alert threshold, and the last AQI was
# at the alerting threshold or below, we won't send the alert.
# For example, if the user sets their threshold at UNHEALTHY, they won't
# be notified when the AQI transitions from UNHEALTHY to UNHEALHY FOR SENSITIVE GROUPS
# or from UNHEALTHY to MODERATE, but will be notified if the AQI transitions from
# VERY UNHEALTHY to UNHEALTHY.
if curr_aqi_level < alert_threshold and last_aqi_level <= alert_threshold:
return False
# If the current AQI is at the alert threshold but the last AQI was under it,
# don't send the alert because we haven't crossed the threshold yet.
if curr_aqi_level == alert_threshold and last_aqi_level < alert_threshold:
return False
# Do not alert clients who received an alert recently unless AQI has changed markedly.
was_alerted_recently = self.last_alert_sent_at > timestamp() - (60 *
60 * 6)
last_aqi = self.get_last_aqi()
if was_alerted_recently and abs(curr_aqi - last_aqi) < 50:
return False
message = gettext(
'AQI is now %(curr_aqi)s in zipcode %(zipcode)s (level: %(curr_aqi_level)s).\n\nReply "M" for Menu or "E" to end alerts.',
zipcode=self.zipcode.zipcode,
curr_aqi_level=curr_aqi_level.display,
curr_aqi=curr_aqi,
)
if not self.send_message(message):
return False
self.last_alert_sent_at = timestamp()
self.last_pm25 = self.zipcode.pm25
self.last_pm_cf_1 = self.zipcode.pm_cf_1
self.last_humidity = self.zipcode.humidity
self.num_alerts_sent += 1
db.session.commit()
self.log_event(EventType.ALERT,
zipcode=self.zipcode.zipcode,
pm25=curr_pm25)
return True
def pm25_stale_cutoff(cls) -> float:
"""Timestamp before which pm25 measurements are considered stale."""
return timestamp() - (60 * 60)
def timestamp(self) -> int:
return timestamp()
