def upload_readings(transforms, meter_oid: int, scraper: str, task_id: str,
readings) -> Status:
updated: List[MeterReading] = []
if readings:
readings = interval_transform.transform(transforms, task_id, scraper,
meter_oid, readings)
log.info("writing interval data to the database for %s %s", scraper,
meter_oid)
updated = MeterReading.merge_readings(
MeterReading.from_json(meter_oid, readings))
if task_id and config.enabled("ES_INDEX_JOBS"):
index.set_interval_fields(task_id, updated)
log.info("Final Interval Summary")
for when, intervals in readings.items():
none_count = sum(1 for x in intervals if x is None)
factor = (24 / len(intervals)) if len(intervals) > 0 else 1.0
kWh = sum(x for x in intervals if x is not None) * factor
log.info("%s: %d intervals. %.1f net kWh, %d null values." %
(when, len(intervals), kWh, none_count))
path = os.path.join(config.WORKING_DIRECTORY, "readings.csv")
with open(path, "w") as csvfile:
writer = csv.writer(csvfile)
writer.writerow(["Service", "Date", "Readings"])
for when, intervals in readings.items():
writer.writerow([meter_oid, str(when)] +
[str(x) for x in intervals])
log.info("Wrote interval data to %s." % path)
if updated:
return Status.SUCCEEDED
return Status.COMPLETED
def test_parse_readings(self):
"""Parse meter readings in scraper format into MeterReading objects."""
dt = (datetime.today() - timedelta(days=3)).date()
# data size doesn't match meter interval
readings: Dict[str, List[float]] = {
dt.strftime("%Y-%m-%d"): [1.0] * 24
}
with self.assertRaises(InvalidMeterDataException):
MeterReading.from_json(self.meter_id, readings)
# values are not floats
readings: Dict[str, List[float]] = {
dt.strftime("%Y-%m-%d"): ["1.0" * 96]
}
with self.assertRaises(InvalidMeterDataException):
MeterReading.from_json(self.meter_id, readings)
# valid data
valid_dates: Set[date] = set()
for idx in range(3):
valid_dates.add(dt)
# Check that both floating point and integer values are acceptable.
readings[dt.strftime("%Y-%m-%d")] = [2.0] * 90 + [2] * 6
dt += timedelta(days=1)
# empty data ignored
readings[dt.strftime("%Y-%m-%d")] = []
readings[(dt + timedelta(days=1)).strftime("%Y-%m-%d")] = [None] * 96
meter_readings: List[MeterReading] = MeterReading.from_json(
self.meter_id, readings)
self.assertEqual(3, len(meter_readings))
for row in meter_readings:
self.assertIn(row.occurred, valid_dates)
self.assertEqual([2.0] * 96, row.readings)
self.assertFalse(row.frozen)
def test_new_readings(self):
"""New readings can be added."""
query = db.session.query(MeterReading).filter_by(meter=self.meter_id)
self.assertEqual(7, query.count(), "7 readings from setup")
dt = (datetime.today() - timedelta(days=3)).date()
start_dt = dt
readings: Dict[str, List[float]] = {}
for idx in range(3):
readings[dt.strftime("%Y-%m-%d")] = [2.0] * 96
dt += timedelta(days=1)
MeterReading.merge_readings(
MeterReading.from_json(self.meter_id, readings))
db.session.flush()
self.assertEqual(10, query.count(), "10 readings after save")
for row in query.filter(MeterReading.occurred < start_dt):
self.assertEqual(
96.0,
sum(row.readings),
"existing readings for %s unchanged" % row.occurred,
)
self.assertEqual(row.occurred, row.modified.date())
for row in query.filter(MeterReading.occurred >= start_dt):
self.assertEqual(96.0 * 2, sum(row.readings),
"new readings added for %s" % row.occurred)
self.assertEqual(date.today(), row.modified.date())
def test_merge_readings(self):
"""New readings will be merged with old readings."""
readings: Dict[str, List[float]] = {}
# replace full row
full_dt = date.today() - timedelta(days=10)
readings[full_dt.strftime("%Y-%m-%d")] = [2.0] * 96
# replace partial row
partial_dt = date.today() - timedelta(days=9)
readings[partial_dt.strftime("%Y-%m-%d")] = [None] * 90 + [2.0] * 6
MeterReading.merge_readings(
MeterReading.from_json(self.meter_id, readings))
db.session.flush()
query = db.session.query(MeterReading).filter_by(meter=self.meter_id)
self.assertEqual(7, query.count(), "7 readings from setup")
for row in query:
if row.occurred == full_dt:
self.assertEqual(96.0 * 2, sum(row.readings),
"%s fully replaced" % row.occurred)
self.assertEqual(date.today(), row.modified.date())
elif row.occurred == partial_dt:
self.assertEqual(
90.0 + 12.0,
sum(row.readings),
"missing values for %s don't replace non-null values" %
row.occurred,
)
self.assertEqual(date.today(), row.modified.date())
else:
self.assertEqual(96.0, sum(row.readings))
self.assertEqual(row.occurred, row.modified.date())
def test_last_reading_date(self):
page = MeterDataPage(None, None)
start_date = date(2020, 5, 1)
updated_start_date = page.start_date_from_readings(
self.meter.oid, start_date)
self.assertEqual(start_date, updated_start_date,
"no readings uses requested start date")
# create a reading record older than start date
reading_date = start_date - timedelta(days=3)
db.session.add(
MeterReading(meter=self.meter.oid,
occurred=reading_date,
readings=[1.0] * 96))
db.session.flush()
updated_start_date = page.start_date_from_readings(
self.meter.oid, start_date)
self.assertEqual(
reading_date,
updated_start_date,
"use oldest reading date when older than start date",
)
reading_date = start_date + timedelta(days=3)
# create a reading record newer than start date
db.session.add(
MeterReading(meter=self.meter.oid,
occurred=reading_date,
readings=[1.0] * 96))
db.session.flush()
updated_start_date = page.start_date_from_readings(
self.meter.oid, start_date)
self.assertEqual(
start_date,
updated_start_date,
"use requested start date when newer readings exist",
)
def test_set_interval_fields(self, index_etl_run):
meter = self.meters[0]
# no readings
task_id = "abc123"
index.set_interval_fields(task_id, [])
self.assertEqual(
{
"updatedDays": 0,
},
index_etl_run.call_args[0][1],
)
# with users and readings
today = date.today()
dates = [
today - timedelta(days=7),
today - timedelta(days=6),
today - timedelta(days=5),
]
readings = []
for dt in dates:
readings.append(
MeterReading(meter=meter.oid, occurred=dt,
readings=[1.0] * 96))
index.set_interval_fields(task_id, readings)
expected = {
"updatedDays": 3,
"intervalFrom": dates[0],
"intervalTo": dates[-1],
"age": 5,
}
self.assertEqual(expected, index_etl_run.call_args[0][1])
def test_frozen_readings(self):
"""New readings will not replace frozen readings."""
query = (db.session.query(MeterReading).filter_by(
meter=self.meter_id).order_by(MeterReading.occurred))
reading = query.first()
reading.frozen = True
# if unchanged, SQLAlchemy will default to now
reading.modified = datetime(reading.occurred.year,
reading.occurred.month,
reading.occurred.day, 1)
db.session.add(reading)
frozen_dt = reading.occurred
dt = reading.occurred
readings: Dict[str, List[float]] = {}
for idx in range(3):
readings[dt.strftime("%Y-%m-%d")] = [2.0] * 96
dt += timedelta(days=1)
latest_dt = dt
MeterReading.merge_readings(
MeterReading.from_json(self.meter_id, readings))
db.session.flush()
# first (frozen) reading unchanged
reading = query.filter(MeterReading.occurred == frozen_dt).first()
self.assertEqual(96.0, sum(reading.readings),
"frozen data not updated")
self.assertEqual(reading.occurred, reading.modified.date(),
"frozen modified not updated")
# other 2 readings updated
for row in query.filter(MeterReading.occurred > reading.occurred):
if row.occurred < latest_dt:
self.assertEqual(
96.0 * 2,
sum(row.readings),
"readings for %s updated" % row.occurred,
)
self.assertEqual(date.today(), row.modified.date())
else:
self.assertEqual(96.0, sum(row.readings),
"readings for %s unchanged" % row.occurred)
self.assertEqual(row.occurred, row.modified.date())
self.assertEqual(7, query.count(), "7 readings from setup")
def setUp(self):
meter = Meter(
commodity="kw",
interval=15,
kind="main",
name="Test Meter 1-%s" % datetime.now().strftime("%s"),
)
db.session.add(meter)
db.session.flush()
self.meter_id = meter.oid
# create readings for a week ago
dt = date.today() - timedelta(days=14)
for idx in range(7):
db.session.add(
MeterReading(
meter=meter.oid,
occurred=dt,
readings=[1.0] * 96,
frozen=False,
modified=datetime(dt.year, dt.month, dt.day),
))
dt += timedelta(days=1)
db.session.flush()