def _get_DataTimeSlots(self, data_id=None, sensor=None, from_dt=None, to_dt=None, timeSlotSpan=None, cached=False, trustme=False): # Initialize cursor cur = self.conn.cursor() # Check for data structure in the db if not trustme: if not self.check_structure_for_DataTimeSlots(sensor, can_initialize=False): raise StorageException('{}: Sorry, DataTimeSlots the structure for the sensor {} is not found.'.format(self.__class__.__name__, sensor)) # Get column names labels = [] for i, item in enumerate(cur.execute('PRAGMA table_info({}_DataTimeSlots);'.format(sensor.__class__.__name__))): if i>1: labels.append(fix_label_from_sqlite(item[1])) # Check that we have every required labels in the storage for label in sensor.Slots_data_labels: if label.lower() not in labels: raise ConsistencyException('Sensor label "{}" not found in {}'.format(label, labels)) # Use the iterator of SQLite (streaming) if from_dt and to_dt: from_s = s_from_dt(from_dt) to_s = s_from_dt(to_dt) query = 'SELECT * from {}_DataTimeSlots where sid="{}" and span="{}" and start_ts >= {} and end_ts <= {}'.format(sensor.__class__.__name__, sensor.id, timeSlotSpan, from_s, to_s) elif (not from_dt and to_dt) or (from_dt and not to_dt): raise InputException('Sorry, please give both from_dt and to_dt or none.') else: # Select all data query = 'SELECT * from {}_DataTimeSlots where sid="{}" and span="{}"'.format(sensor.__class__.__name__, sensor.id, timeSlotSpan) # Create the DataStream dataTimeStream = SQLiteDataTimeStream(cur=cur, query=query, sensor=sensor, data_type=DataTimeSlot, labels=labels, timeSlotSpan=timeSlotSpan) # Create a StreamingDataTimeSeries with the above DataStream stramingDataTimeSeries = StreamingDataTimeSeries(dataTimeStream=dataTimeStream, cached=cached) return stramingDataTimeSeries
def aggregate(self, dataTimeSeries, start_dt, end_dt, timeSlotSpan, allow_None_data=False): #------------------- # Sanity checks #------------------- # First of all Ensure we are operating on PhysicalData (so we eill have PhysicalQuantities), # otherwise raise an error: if self.Sensor.Points_type.data_type != PhysicalData: raise NotImplementedError('Sorry, only PhysicalData data type is supported for now. Adding support for generic data is not too much complicated anyway') #------------------- # Support vars #------------------- Slot_data_labels_to_generate = self.Sensor.Slots_data_labels Slot_data_labels = [] # TODO: REMOVE, this is the same as Slot_data_labels_to_generate Slot_data_values = [] # Create start and end Points. TODO: is this not performant, also with the time zone? start_Point = TimePoint(t=s_from_dt(start_dt), tz=start_dt.tzinfo) end_Point = TimePoint(t=s_from_dt(end_dt), tz=start_dt.tzinfo) #------------------- # Compute coverage #------------------- logger.debug(' Now computing coverage...') Slot_coverage = compute_1D_coverage(dataSeries = dataTimeSeries, start_Point = start_Point, end_Point = end_Point) # If no coverage return list of None if Slot_coverage == 0.0: if allow_None_data: Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels_to_generate, values = [None for _ in Slot_data_labels_to_generate], # Force "trustme" to allow None in data trustme = True) dataTimeSlot = self.Sensor.Slots_type(start = start_Point, end = end_Point, data = Slot_physicalData, span = timeSlotSpan, coverage = 0.0) logger.info('Done aggregating, slot: %s', dataTimeSlot) return dataTimeSlot else: raise NoDataException('This slot has coverage of 0.0, cannot compute any data! (start={}, end={})'.format(start_Point, end_Point)) #-------------------------------------------- # Understand the labels to produce and to # operate on according to the sensor type #-------------------------------------------- # TODO: define a mapping somwhere and perform this operations only once. for Slot_data_label_to_generate in Slot_data_labels_to_generate: handled = False Generator = None Operation = None operate_on = None # Labels could already be PhysicalQuantiy objects if not isinstance(Slot_data_label_to_generate, PhysicalQuantity): physicalQuantity_to_generate = PhysicalQuantity(Slot_data_label_to_generate) if physicalQuantity_to_generate.op is None: raise ConfigurationException('Sorry, PhysicalQuantity "{}" has no operation defined, cannot aggregate.'.format(physicalQuantity_to_generate)) #-------------------------------------- # Handle generators #-------------------------------------- # Is this physicalQuantity generated by a custom generator defined inside the sensor class? try: # TODO: Use the 'provides' logic Generator = getattr(self.Sensor, Slot_data_label_to_generate) handled = True except AttributeError: pass # Is this physicalQuantity generated by a standard generator? try: from luna.aggregators import generators Generator = getattr(generators, Slot_data_label_to_generate) handled = True except AttributeError: pass #-------------------------------------- # Handle operations #-------------------------------------- # Is this physicalQuantity_to_generate generated by applying the operation to another # physicalQuantity_to_generate defined in the Points? for Point_physicalQuantity in self.Sensor.Points_data_labels: # Standard operation if physicalQuantity_to_generate.name_unit == Point_physicalQuantity: try: from luna.aggregators import operations Operation = getattr(operations, physicalQuantity_to_generate.op) except AttributeError: # TODO: add more info (i.e. sensor class etc?) raise ConfigurationException('Sorry, I cannot find any valid operation for {} in luna.aggregators.operations'.format(physicalQuantity_to_generate.op)) operate_on = Point_physicalQuantity handled = True break logger.debug('For generating %s I will use generator %s and operation %s', physicalQuantity_to_generate, Generator, Operation) #---------------------- # Now compute #---------------------- if not handled: # TODO: add more info (i.e. sensor class etc?) raise ConfigurationException('Could not handle "{}", as I did not find any way to generate it. Please check your configuration for this sensor'.format(Slot_data_label_to_generate)) # Set if streaming Operation/generator: try: Generator_is_streaming = Generator.is_stremaing except AttributeError: Generator_is_streaming = False try: Operation_is_streaming = Operation.is_stremaing except AttributeError: Operation_is_streaming = False if Generator and not Generator_is_streaming: logger.debug('Running generator %s on to generate %s', Generator, Slot_data_label_to_generate) # A generator also requires access to the aggregated data, so we initialize it also here** Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels, values = Slot_data_values) # Run the generator result = Generator.generate(dataSeries = dataTimeSeries, start_Point = start_Point, end_Point = end_Point, aggregated_data = Slot_physicalData) # Ok, append the operation/generator results to the labels and values logger.debug('Done running generator') Slot_data_labels.append(Slot_data_label_to_generate) Slot_data_values.append(result) elif Operation and not Operation_is_streaming: logger.debug('Running operation %s on %s to generate %s', Operation, operate_on, Slot_data_label_to_generate) # Run the operation result = Operation.compute_on_Points(dataSeries = dataTimeSeries.lazy_filter_data_label(label=operate_on), start_Point = start_Point, end_Point = end_Point) # Ok, append the operation/generator results to the labels and values logger.debug('Done running operation') Slot_data_labels.append(Slot_data_label_to_generate) Slot_data_values.append(result) else: raise ConsistencyException('No generator nor Operation?! (maybe got streaming which is not yet supported)') #---------------------- # Build results #---------------------- Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels, values = Slot_data_values) dataTimeSlot = self.Sensor.Slots_type(start = start_Point, end = end_Point, data = Slot_physicalData, span = timeSlotSpan, coverage = Slot_coverage) # Return results logger.info('Done aggregating, slot: %s', dataTimeSlot) return dataTimeSlot