def test_existing_meter_reading_has_reading_source_unit_and_conversion_factor_updated_if_import_file_references_previous_entry(
self):
property = Property.objects.get(pk=self.property_1.id)
# Create a meter with the same details of one meter in the import file
unsaved_meter = Meter(
property=property,
source=Meter.GREENBUTTON,
source_id=
'User/6150855/UsagePoint/409483/MeterReading/1/IntervalBlock/1',
type=Meter.ELECTRICITY_GRID,
)
unsaved_meter.save()
existing_meter = Meter.objects.get(pk=unsaved_meter.id)
# Create a reading with the same date as one from the import file but different reading
start_time = make_aware(datetime(2011, 3, 5, 21, 0, 0),
timezone=self.tz_obj)
end_time = make_aware(datetime(2011, 3, 5, 21, 15, 0),
timezone=self.tz_obj)
unsaved_meter_reading = MeterReading(meter=existing_meter,
start_time=start_time,
end_time=end_time,
reading=1000,
source_unit="GJ",
conversion_factor=947.82)
unsaved_meter_reading.save()
url = reverse("api:v2:import_files-save-raw-data",
args=[self.import_file.id])
post_params = {
'cycle_id': self.cycle.pk,
'organization_id': self.org.pk,
}
self.client.post(url, post_params)
# Just as in the first test, 2 meter readings should exist
self.assertEqual(MeterReading.objects.all().count(), 2)
refreshed_property = Property.objects.get(pk=self.property_1.id)
refreshed_meter = refreshed_property.meters.get(
type=Meter.ELECTRICITY_GRID)
meter_reading = refreshed_meter.meter_readings.get(
start_time=start_time)
self.assertEqual(meter_reading.end_time, end_time)
self.assertEqual(meter_reading.reading, 1790 * 3.41 / 1000)
self.assertEqual(meter_reading.source_unit,
'kWh (thousand Watt-hours)')
self.assertEqual(meter_reading.conversion_factor, 3.41)
def test_existing_meter_reading_has_reading_source_unit_and_conversion_factor_updated_if_import_file_references_previous_entry(
self):
property = Property.objects.get(pk=self.property_1.id)
# Create a meter with the same details of one meter in the import file
unsaved_meter = Meter(
property=property,
source=Meter.PORTFOLIO_MANAGER,
source_id='5766973-0',
type=Meter.ELECTRICITY_GRID,
)
unsaved_meter.save()
existing_meter = Meter.objects.get(pk=unsaved_meter.id)
# Create a reading with the same date as one from the import file but different reading
start_time = make_aware(datetime(2016, 1, 1, 0, 0, 0),
timezone=self.tz_obj)
end_time = make_aware(datetime(2016, 2, 1, 0, 0, 0),
timezone=self.tz_obj)
unsaved_meter_reading = MeterReading(meter=existing_meter,
start_time=start_time,
end_time=end_time,
reading=12345,
source_unit="GJ",
conversion_factor=947.82)
unsaved_meter_reading.save()
url = reverse("api:v3:import_files-start-save-data",
args=[self.import_file.id])
url += f'?organization_id={self.org.pk}'
post_params = {
'cycle_id': self.cycle.pk,
}
self.client.post(url, post_params)
# Just as in the first test, 8 meter readings should exist
self.assertEqual(MeterReading.objects.all().count(), 8)
refreshed_property = Property.objects.get(pk=self.property_1.id)
refreshed_meter = refreshed_property.meters.get(
type=Meter.ELECTRICITY_GRID)
meter_reading = refreshed_meter.meter_readings.get(
start_time=start_time)
self.assertEqual(meter_reading.end_time, end_time)
self.assertEqual(meter_reading.reading, 597478.9)
self.assertEqual(meter_reading.source_unit, "kBtu (thousand Btu)")
self.assertEqual(meter_reading.conversion_factor, 1)
def test_existing_meter_is_found_and_used_if_import_file_should_reference_it(
self):
property = Property.objects.get(pk=self.property_1.id)
# Create a meter with the same details of the meter in the import file
unsaved_meter = Meter(
property=property,
source=Meter.GREENBUTTON,
source_id=
'User/6150855/UsagePoint/409483/MeterReading/1/IntervalBlock/1',
type=Meter.ELECTRICITY_GRID,
)
unsaved_meter.save()
existing_meter = Meter.objects.get(pk=unsaved_meter.id)
# Create a reading with a different date from those in the import file
unsaved_meter_reading = MeterReading(
meter=existing_meter,
start_time=make_aware(datetime(2018, 1, 1, 0, 0, 0),
timezone=self.tz_obj),
end_time=make_aware(datetime(2018, 2, 1, 0, 0, 0),
timezone=self.tz_obj),
reading=12345,
)
unsaved_meter_reading.save()
existing_meter_reading = MeterReading.objects.get(reading=12345)
url = reverse("api:v2:import_files-save-raw-data",
args=[self.import_file.id])
post_params = {
'cycle_id': self.cycle.pk,
'organization_id': self.org.pk,
}
self.client.post(url, post_params)
refreshed_property_1 = Property.objects.get(pk=self.property_1.id)
self.assertEqual(refreshed_property_1.meters.all().count(), 1)
refreshed_meter = refreshed_property_1.meters.get(
type=Meter.ELECTRICITY_GRID)
meter_reading_10, meter_reading_11, meter_reading_12 = list(
refreshed_meter.meter_readings.order_by('start_time').all())
self.assertEqual(meter_reading_10.reading, 1790 * 3.41 / 1000)
self.assertEqual(meter_reading_11.reading, 1791 * 3.41 / 1000)
# Sanity check to be sure, nothing was changed with existing meter reading
self.assertEqual(meter_reading_12, existing_meter_reading)
def test_existing_meter_is_found_and_used_if_import_file_should_reference_it(
self):
property = Property.objects.get(pk=self.property_1.id)
# Create a meter with the same details of one meter in the import file
unsaved_meter = Meter(
property=property,
source=Meter.PORTFOLIO_MANAGER,
source_id='5766973-0',
type=Meter.ELECTRICITY_GRID,
)
unsaved_meter.save()
existing_meter = Meter.objects.get(pk=unsaved_meter.id)
# Create a reading with a different date from those in the import file
unsaved_meter_reading = MeterReading(
meter=existing_meter,
start_time=make_aware(datetime(2018, 1, 1, 0, 0, 0),
timezone=self.tz_obj),
end_time=make_aware(datetime(2018, 2, 1, 0, 0, 0),
timezone=self.tz_obj),
reading=12345,
conversion_factor=1.0)
unsaved_meter_reading.save()
existing_meter_reading = MeterReading.objects.get(reading=12345)
url = reverse("api:v3:import_files-start-save-data",
args=[self.import_file.id])
url += f'?organization_id={self.org.pk}'
post_params = {
'cycle_id': self.cycle.pk,
}
self.client.post(url, post_params)
refreshed_property_1 = Property.objects.get(pk=self.property_1.id)
self.assertEqual(refreshed_property_1.meters.all().count(), 2)
refreshed_meter = refreshed_property_1.meters.get(
type=Meter.ELECTRICITY_GRID)
meter_reading_10, meter_reading_11, meter_reading_12 = list(
refreshed_meter.meter_readings.order_by('start_time').all())
self.assertEqual(meter_reading_10.reading, 597478.9)
self.assertEqual(meter_reading_11.reading, 548603.7)
# Sanity check to be sure, nothing was changed with existing meter reading
self.assertEqual(meter_reading_12, existing_meter_reading)
def data_analyse(ts_data, name):
finer_ts = []
monthly_ts = []
cache = {}
today_date = date.today()
today_str = today_date.strftime("%m/%d/%Y")
today_month = int(today_date.month)
today_year = int(today_date.year)
immediate_aggregate = False
for ts_cell in ts_data:
if name == "Energy Template" or name == "PM":
# convert to seconds
ts_cell["start"] = int(ts_cell["start"]) / 1000
ts_cell["interval"] = int(ts_cell["interval"]) / 1000
try:
ts_cell["canonical_id"] = str(int(float(ts_cell["canonical_id"])))
except ValueError:
continue
custom_meter_id = ts_cell["custom_meter_id"]
try:
ts_cell["custom_meter_id"] = str(int(float(ts_cell["custom_meter_id"])))
except ValueError:
ts_cell["custom_meter_id"] = custom_meter_id
interval = int(ts_cell["interval"])
building_id = str(ts_cell["canonical_id"])
custom_meter_id = str(ts_cell["custom_meter_id"])
if interval < interval_threshold:
ts_cell["insert_date"] = today_str
ts_dateObj = get_month_from_ts(ts_cell["start"])
if ts_dateObj["month"] != today_month or ts_dateObj["year"] != today_year:
# has back filling
immediate_aggregate = True
finer_ts.append(ts_cell)
else:
monthly_ts.append(ts_cell)
# create or retrieve seed_meter_id
if not building_id + "_" + custom_meter_id in cache:
res = (
Meter.objects.filter(custom_meter_id=custom_meter_id)
.select_related()
.filter(canonical_building=building_id)
)
if not res:
# create new meter record
new_meter = Meter(
name=(name + " METER"),
energy_type=ts_cell["energy_type_int"],
energy_units=ts_cell["uom_int"],
custom_meter_id=ts_cell["custom_meter_id"],
)
new_meter.save()
new_meter.canonical_building.add(CanonicalBuilding.objects.get(id=building_id))
seed_meter_id = int(new_meter.id)
else:
seed_meter_id = int(res[0].id)
cache[building_id + "_" + custom_meter_id] = seed_meter_id
for ts_cell in monthly_ts:
building_id = str(ts_cell["canonical_id"])
custom_meter_id = str(ts_cell["custom_meter_id"])
seed_meter_id = int(cache[building_id + "_" + custom_meter_id])
# save record to timeseries table
begin_ts = int(ts_cell["start"])
interval = int(ts_cell["interval"])
new_ts = TimeSeries(
begin_time=datetime.fromtimestamp(begin_ts),
end_time=datetime.fromtimestamp(begin_ts + interval),
reading=float(ts_cell["value"]),
meter_id=seed_meter_id,
)
new_ts.save()
_log.info("insert monthly data into postgresql finished")
insert_flag = tsdb.insert(finer_ts)
_log.info("insert ts data into KairosDB finished: " + str(insert_flag))
if insert_flag and immediate_aggregate:
_log.info("Having back filling data, aggregate immediately")
tasks.aggregate_monthly_data(ts_data[0]["canonical_id"])
_log.info("Immediate aggregation finished")