def aggregate(self, dataTimeSeries, start_dt, end_dt, timeSlotSpan, allow_None_data=False): #------------------- # Sanity checks #------------------- # First of all Ensure we are operating on PhysicalData (so we eill have PhysicalQuantities), # otherwise raise an error: if self.Sensor.Points_type.data_type != PhysicalData: raise NotImplementedError('Sorry, only PhysicalData data type is supported for now. Adding support for generic data is not too much complicated anyway') #------------------- # Support vars #------------------- Slot_data_labels_to_generate = self.Sensor.Slots_data_labels Slot_data_labels = [] # TODO: REMOVE, this is the same as Slot_data_labels_to_generate Slot_data_values = [] # Create start and end Points. TODO: is this not performant, also with the time zone? start_Point = TimePoint(t=s_from_dt(start_dt), tz=start_dt.tzinfo) end_Point = TimePoint(t=s_from_dt(end_dt), tz=start_dt.tzinfo) #------------------- # Compute coverage #------------------- logger.debug(' Now computing coverage...') Slot_coverage = compute_1D_coverage(dataSeries = dataTimeSeries, start_Point = start_Point, end_Point = end_Point) # If no coverage return list of None if Slot_coverage == 0.0: if allow_None_data: Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels_to_generate, values = [None for _ in Slot_data_labels_to_generate], # Force "trustme" to allow None in data trustme = True) dataTimeSlot = self.Sensor.Slots_type(start = start_Point, end = end_Point, data = Slot_physicalData, span = timeSlotSpan, coverage = 0.0) logger.info('Done aggregating, slot: %s', dataTimeSlot) return dataTimeSlot else: raise NoDataException('This slot has coverage of 0.0, cannot compute any data! (start={}, end={})'.format(start_Point, end_Point)) #-------------------------------------------- # Understand the labels to produce and to # operate on according to the sensor type #-------------------------------------------- # TODO: define a mapping somwhere and perform this operations only once. for Slot_data_label_to_generate in Slot_data_labels_to_generate: handled = False Generator = None Operation = None operate_on = None # Labels could already be PhysicalQuantiy objects if not isinstance(Slot_data_label_to_generate, PhysicalQuantity): physicalQuantity_to_generate = PhysicalQuantity(Slot_data_label_to_generate) if physicalQuantity_to_generate.op is None: raise ConfigurationException('Sorry, PhysicalQuantity "{}" has no operation defined, cannot aggregate.'.format(physicalQuantity_to_generate)) #-------------------------------------- # Handle generators #-------------------------------------- # Is this physicalQuantity generated by a custom generator defined inside the sensor class? try: # TODO: Use the 'provides' logic Generator = getattr(self.Sensor, Slot_data_label_to_generate) handled = True except AttributeError: pass # Is this physicalQuantity generated by a standard generator? try: from luna.aggregators import generators Generator = getattr(generators, Slot_data_label_to_generate) handled = True except AttributeError: pass #-------------------------------------- # Handle operations #-------------------------------------- # Is this physicalQuantity_to_generate generated by applying the operation to another # physicalQuantity_to_generate defined in the Points? for Point_physicalQuantity in self.Sensor.Points_data_labels: # Standard operation if physicalQuantity_to_generate.name_unit == Point_physicalQuantity: try: from luna.aggregators import operations Operation = getattr(operations, physicalQuantity_to_generate.op) except AttributeError: # TODO: add more info (i.e. sensor class etc?) raise ConfigurationException('Sorry, I cannot find any valid operation for {} in luna.aggregators.operations'.format(physicalQuantity_to_generate.op)) operate_on = Point_physicalQuantity handled = True break logger.debug('For generating %s I will use generator %s and operation %s', physicalQuantity_to_generate, Generator, Operation) #---------------------- # Now compute #---------------------- if not handled: # TODO: add more info (i.e. sensor class etc?) raise ConfigurationException('Could not handle "{}", as I did not find any way to generate it. Please check your configuration for this sensor'.format(Slot_data_label_to_generate)) # Set if streaming Operation/generator: try: Generator_is_streaming = Generator.is_stremaing except AttributeError: Generator_is_streaming = False try: Operation_is_streaming = Operation.is_stremaing except AttributeError: Operation_is_streaming = False if Generator and not Generator_is_streaming: logger.debug('Running generator %s on to generate %s', Generator, Slot_data_label_to_generate) # A generator also requires access to the aggregated data, so we initialize it also here** Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels, values = Slot_data_values) # Run the generator result = Generator.generate(dataSeries = dataTimeSeries, start_Point = start_Point, end_Point = end_Point, aggregated_data = Slot_physicalData) # Ok, append the operation/generator results to the labels and values logger.debug('Done running generator') Slot_data_labels.append(Slot_data_label_to_generate) Slot_data_values.append(result) elif Operation and not Operation_is_streaming: logger.debug('Running operation %s on %s to generate %s', Operation, operate_on, Slot_data_label_to_generate) # Run the operation result = Operation.compute_on_Points(dataSeries = dataTimeSeries.lazy_filter_data_label(label=operate_on), start_Point = start_Point, end_Point = end_Point) # Ok, append the operation/generator results to the labels and values logger.debug('Done running operation') Slot_data_labels.append(Slot_data_label_to_generate) Slot_data_values.append(result) else: raise ConsistencyException('No generator nor Operation?! (maybe got streaming which is not yet supported)') #---------------------- # Build results #---------------------- Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels, values = Slot_data_values) dataTimeSlot = self.Sensor.Slots_type(start = start_Point, end = end_Point, data = Slot_physicalData, span = timeSlotSpan, coverage = Slot_coverage) # Return results logger.info('Done aggregating, slot: %s', dataTimeSlot) return dataTimeSlot
def test_compute_1D_coverage_basic(self): # ---------------------------- # Test wrong init parameters # ---------------------------- with self.assertRaises(InputException): compute_1D_coverage(dataSeries=None, start_Point=None, end_Point=None) with self.assertRaises(NotImplementedError): compute_1D_coverage(dataSeries=self.dataTimeSeries1, start_Point=None, end_Point=None) with self.assertRaises(InputException): compute_1D_coverage(dataSeries=self.dataTimeSeries1, start_Point=5, end_Point=TimePoint(t=3)) with self.assertRaises(InputException): compute_1D_coverage(dataSeries=self.dataTimeSeries1, start_Point=TimePoint(t=3), end_Point=5) # ---------------------------- # Test logic # ---------------------------- # Full coverage (coverage=1.0) start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00 end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00 Slot_coverage = compute_1D_coverage( dataSeries=self.dataTimeSeries1, start_Point=start_Point, end_Point=end_Point ) self.assertEqual(Slot_coverage, 1.0) # A) Full coverage (coverage=1.0) again, to test repeatability start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00 end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00 Slot_coverage = compute_1D_coverage( dataSeries=self.dataTimeSeries1, start_Point=start_Point, end_Point=end_Point ) self.assertEqual(Slot_coverage, 1.0) # B) Full coverage (coverage=1.0) witjout prev/next in the timeSeries start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00 end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00 Slot_coverage = compute_1D_coverage( dataSeries=self.dataTimeSeries2, start_Point=start_Point, end_Point=end_Point ) self.assertEqual(Slot_coverage, 1.0) # C) Missing ten minutes over 30 at the end (coverage=0.683)) start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00 end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00 Slot_coverage = compute_1D_coverage( dataSeries=self.dataTimeSeries3, start_Point=start_Point, end_Point=end_Point ) # 20 minutes plus other 30 secs validity for the 20th point over 30 minutes self.assertEqual(Slot_coverage, (((20 * 60.0) + 30.0) / (30 * 60.0))) # D) Missing ten minutes over 30 at the beginning (coverage=0.683) start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00 end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00 Slot_coverage = compute_1D_coverage( dataSeries=self.dataTimeSeries4, start_Point=start_Point, end_Point=end_Point ) # 20 minutes plus other 30 secs (previous) validity for the 10th point over 30 minutes self.assertEqual(Slot_coverage, (((20 * 60.0) + 30.0) / (30 * 60.0))) # E) Missing eleven minutes over 30 in the middle (coverage=0.66) start_Point = TimePoint(t=1436022000, tz="Europe/Rome") # 2015-07-04 17:00:00+02:00 end_Point = TimePoint(t=1436022000 + 1800, tz="Europe/Rome") # 2015-07-04 17:30:00+02:00 Slot_coverage = compute_1D_coverage( dataSeries=self.dataTimeSeries5, start_Point=start_Point, end_Point=end_Point ) # 20 minutes plus other 30 secs (previous) validity for the 10th point over 30 minutes self.assertAlmostEqual(Slot_coverage, (2.0 / 3.0))