def aggregate(self, dataTimeSeries, start_dt, end_dt, timeSlotSpan, allow_None_data=False):
#-------------------
# Sanity checks
#-------------------
# First of all Ensure we are operating on PhysicalData (so we eill have PhysicalQuantities),
# otherwise raise an error:
if self.Sensor.Points_type.data_type != PhysicalData:
raise NotImplementedError('Sorry, only PhysicalData data type is supported for now. Adding support for generic data is not too much complicated anyway')
#-------------------
# Support vars
#-------------------
Slot_data_labels_to_generate = self.Sensor.Slots_data_labels
Slot_data_labels = [] # TODO: REMOVE, this is the same as Slot_data_labels_to_generate
Slot_data_values = []
# Create start and end Points. TODO: is this not performant, also with the time zone?
start_Point = TimePoint(t=s_from_dt(start_dt), tz=start_dt.tzinfo)
end_Point = TimePoint(t=s_from_dt(end_dt), tz=start_dt.tzinfo)
#-------------------
# Compute coverage
#-------------------
logger.debug(' Now computing coverage...')
Slot_coverage = compute_1D_coverage(dataSeries = dataTimeSeries,
start_Point = start_Point,
end_Point = end_Point)
# If no coverage return list of None
if Slot_coverage == 0.0:
if allow_None_data:
Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels_to_generate,
values = [None for _ in Slot_data_labels_to_generate], # Force "trustme" to allow None in data
trustme = True)
dataTimeSlot = self.Sensor.Slots_type(start = start_Point,
end = end_Point,
data = Slot_physicalData,
span = timeSlotSpan,
coverage = 0.0)
logger.info('Done aggregating, slot: %s', dataTimeSlot)
return dataTimeSlot
else:
raise NoDataException('This slot has coverage of 0.0, cannot compute any data! (start={}, end={})'.format(start_Point, end_Point))
#--------------------------------------------
# Understand the labels to produce and to
# operate on according to the sensor type
#--------------------------------------------
# TODO: define a mapping somwhere and perform this operations only once.
for Slot_data_label_to_generate in Slot_data_labels_to_generate:
handled = False
Generator = None
Operation = None
operate_on = None
# Labels could already be PhysicalQuantiy objects
if not isinstance(Slot_data_label_to_generate, PhysicalQuantity):
physicalQuantity_to_generate = PhysicalQuantity(Slot_data_label_to_generate)
if physicalQuantity_to_generate.op is None:
raise ConfigurationException('Sorry, PhysicalQuantity "{}" has no operation defined, cannot aggregate.'.format(physicalQuantity_to_generate))
#--------------------------------------
# Handle generators
#--------------------------------------
# Is this physicalQuantity generated by a custom generator defined inside the sensor class?
try:
# TODO: Use the 'provides' logic
Generator = getattr(self.Sensor, Slot_data_label_to_generate)
handled = True
except AttributeError:
pass
# Is this physicalQuantity generated by a standard generator?
try:
from luna.aggregators import generators
Generator = getattr(generators, Slot_data_label_to_generate)
handled = True
except AttributeError:
pass
#--------------------------------------
# Handle operations
#--------------------------------------
# Is this physicalQuantity_to_generate generated by applying the operation to another
# physicalQuantity_to_generate defined in the Points?
for Point_physicalQuantity in self.Sensor.Points_data_labels:
# Standard operation
if physicalQuantity_to_generate.name_unit == Point_physicalQuantity:
try:
from luna.aggregators import operations
Operation = getattr(operations, physicalQuantity_to_generate.op)
except AttributeError:
# TODO: add more info (i.e. sensor class etc?)
raise ConfigurationException('Sorry, I cannot find any valid operation for {} in luna.aggregators.operations'.format(physicalQuantity_to_generate.op))
operate_on = Point_physicalQuantity
handled = True
break
logger.debug('For generating %s I will use generator %s and operation %s', physicalQuantity_to_generate, Generator, Operation)
#----------------------
# Now compute
#----------------------
if not handled:
# TODO: add more info (i.e. sensor class etc?)
raise ConfigurationException('Could not handle "{}", as I did not find any way to generate it. Please check your configuration for this sensor'.format(Slot_data_label_to_generate))
# Set if streaming Operation/generator:
try:
Generator_is_streaming = Generator.is_stremaing
except AttributeError:
Generator_is_streaming = False
try:
Operation_is_streaming = Operation.is_stremaing
except AttributeError:
Operation_is_streaming = False
if Generator and not Generator_is_streaming:
logger.debug('Running generator %s on to generate %s', Generator, Slot_data_label_to_generate)
# A generator also requires access to the aggregated data, so we initialize it also here**
Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels,
values = Slot_data_values)
# Run the generator
result = Generator.generate(dataSeries = dataTimeSeries,
start_Point = start_Point,
end_Point = end_Point,
aggregated_data = Slot_physicalData)
# Ok, append the operation/generator results to the labels and values
logger.debug('Done running generator')
Slot_data_labels.append(Slot_data_label_to_generate)
Slot_data_values.append(result)
elif Operation and not Operation_is_streaming:
logger.debug('Running operation %s on %s to generate %s', Operation, operate_on, Slot_data_label_to_generate)
# Run the operation
result = Operation.compute_on_Points(dataSeries = dataTimeSeries.lazy_filter_data_label(label=operate_on),
start_Point = start_Point,
end_Point = end_Point)
# Ok, append the operation/generator results to the labels and values
logger.debug('Done running operation')
Slot_data_labels.append(Slot_data_label_to_generate)
Slot_data_values.append(result)
else:
raise ConsistencyException('No generator nor Operation?! (maybe got streaming which is not yet supported)')
#----------------------
# Build results
#----------------------
Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels,
values = Slot_data_values)
dataTimeSlot = self.Sensor.Slots_type(start = start_Point,
end = end_Point,
data = Slot_physicalData,
span = timeSlotSpan,
coverage = Slot_coverage)
# Return results
logger.info('Done aggregating, slot: %s', dataTimeSlot)
return dataTimeSlot
def test_compute_1D_coverage_basic(self):
# ----------------------------
# Test wrong init parameters
# ----------------------------
with self.assertRaises(InputException):
compute_1D_coverage(dataSeries=None, start_Point=None, end_Point=None)
with self.assertRaises(NotImplementedError):
compute_1D_coverage(dataSeries=self.dataTimeSeries1, start_Point=None, end_Point=None)
with self.assertRaises(InputException):
compute_1D_coverage(dataSeries=self.dataTimeSeries1, start_Point=5, end_Point=TimePoint(t=3))
with self.assertRaises(InputException):
compute_1D_coverage(dataSeries=self.dataTimeSeries1, start_Point=TimePoint(t=3), end_Point=5)
# ----------------------------
# Test logic
# ----------------------------
# Full coverage (coverage=1.0)
start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00
end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00
Slot_coverage = compute_1D_coverage(
dataSeries=self.dataTimeSeries1, start_Point=start_Point, end_Point=end_Point
)
self.assertEqual(Slot_coverage, 1.0)
# A) Full coverage (coverage=1.0) again, to test repeatability
start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00
end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00
Slot_coverage = compute_1D_coverage(
dataSeries=self.dataTimeSeries1, start_Point=start_Point, end_Point=end_Point
)
self.assertEqual(Slot_coverage, 1.0)
# B) Full coverage (coverage=1.0) witjout prev/next in the timeSeries
start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00
end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00
Slot_coverage = compute_1D_coverage(
dataSeries=self.dataTimeSeries2, start_Point=start_Point, end_Point=end_Point
)
self.assertEqual(Slot_coverage, 1.0)
# C) Missing ten minutes over 30 at the end (coverage=0.683))
start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00
end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00
Slot_coverage = compute_1D_coverage(
dataSeries=self.dataTimeSeries3, start_Point=start_Point, end_Point=end_Point
)
# 20 minutes plus other 30 secs validity for the 20th point over 30 minutes
self.assertEqual(Slot_coverage, (((20 * 60.0) + 30.0) / (30 * 60.0)))
# D) Missing ten minutes over 30 at the beginning (coverage=0.683)
start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00
end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00
Slot_coverage = compute_1D_coverage(
dataSeries=self.dataTimeSeries4, start_Point=start_Point, end_Point=end_Point
)
# 20 minutes plus other 30 secs (previous) validity for the 10th point over 30 minutes
self.assertEqual(Slot_coverage, (((20 * 60.0) + 30.0) / (30 * 60.0)))
# E) Missing eleven minutes over 30 in the middle (coverage=0.66)
start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00
end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00
Slot_coverage = compute_1D_coverage(
dataSeries=self.dataTimeSeries5, start_Point=start_Point, end_Point=end_Point
)
# 20 minutes plus other 30 secs (previous) validity for the 10th point over 30 minutes
self.assertAlmostEqual(Slot_coverage, (2.0 / 3.0))