def _get_DataTimeSlots(self, data_id=None, sensor=None, from_dt=None, to_dt=None, timeSlotSpan=None, cached=False, trustme=False):
# Initialize cursor
cur = self.conn.cursor()
# Check for data structure in the db
if not trustme:
if not self.check_structure_for_DataTimeSlots(sensor, can_initialize=False):
raise StorageException('{}: Sorry, DataTimeSlots the structure for the sensor {} is not found.'.format(self.__class__.__name__, sensor))
# Get column names
labels = []
for i, item in enumerate(cur.execute('PRAGMA table_info({}_DataTimeSlots);'.format(sensor.__class__.__name__))):
if i>1:
labels.append(fix_label_from_sqlite(item[1]))
# Check that we have every required labels in the storage
for label in sensor.Slots_data_labels:
if label.lower() not in labels:
raise ConsistencyException('Sensor label "{}" not found in {}'.format(label, labels))
# Use the iterator of SQLite (streaming)
if from_dt and to_dt:
from_s = s_from_dt(from_dt)
to_s = s_from_dt(to_dt)
query = 'SELECT * from {}_DataTimeSlots where sid="{}" and span="{}" and start_ts >= {} and end_ts <= {}'.format(sensor.__class__.__name__, sensor.id, timeSlotSpan, from_s, to_s)
elif (not from_dt and to_dt) or (from_dt and not to_dt):
raise InputException('Sorry, please give both from_dt and to_dt or none.')
else:
# Select all data
query = 'SELECT * from {}_DataTimeSlots where sid="{}" and span="{}"'.format(sensor.__class__.__name__, sensor.id, timeSlotSpan)
# Create the DataStream
dataTimeStream = SQLiteDataTimeStream(cur=cur, query=query, sensor=sensor, data_type=DataTimeSlot, labels=labels, timeSlotSpan=timeSlotSpan)
# Create a StreamingDataTimeSeries with the above DataStream
stramingDataTimeSeries = StreamingDataTimeSeries(dataTimeStream=dataTimeStream, cached=cached)
return stramingDataTimeSeries
def aggregate(self, dataTimeSeries, start_dt, end_dt, timeSlotSpan, allow_None_data=False):
#-------------------
# Sanity checks
#-------------------
# First of all Ensure we are operating on PhysicalData (so we eill have PhysicalQuantities),
# otherwise raise an error:
if self.Sensor.Points_type.data_type != PhysicalData:
raise NotImplementedError('Sorry, only PhysicalData data type is supported for now. Adding support for generic data is not too much complicated anyway')
#-------------------
# Support vars
#-------------------
Slot_data_labels_to_generate = self.Sensor.Slots_data_labels
Slot_data_labels = [] # TODO: REMOVE, this is the same as Slot_data_labels_to_generate
Slot_data_values = []
# Create start and end Points. TODO: is this not performant, also with the time zone?
start_Point = TimePoint(t=s_from_dt(start_dt), tz=start_dt.tzinfo)
end_Point = TimePoint(t=s_from_dt(end_dt), tz=start_dt.tzinfo)
#-------------------
# Compute coverage
#-------------------
logger.debug(' Now computing coverage...')
Slot_coverage = compute_1D_coverage(dataSeries = dataTimeSeries,
start_Point = start_Point,
end_Point = end_Point)
# If no coverage return list of None
if Slot_coverage == 0.0:
if allow_None_data:
Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels_to_generate,
values = [None for _ in Slot_data_labels_to_generate], # Force "trustme" to allow None in data
trustme = True)
dataTimeSlot = self.Sensor.Slots_type(start = start_Point,
end = end_Point,
data = Slot_physicalData,
span = timeSlotSpan,
coverage = 0.0)
logger.info('Done aggregating, slot: %s', dataTimeSlot)
return dataTimeSlot
else:
raise NoDataException('This slot has coverage of 0.0, cannot compute any data! (start={}, end={})'.format(start_Point, end_Point))
#--------------------------------------------
# Understand the labels to produce and to
# operate on according to the sensor type
#--------------------------------------------
# TODO: define a mapping somwhere and perform this operations only once.
for Slot_data_label_to_generate in Slot_data_labels_to_generate:
handled = False
Generator = None
Operation = None
operate_on = None
# Labels could already be PhysicalQuantiy objects
if not isinstance(Slot_data_label_to_generate, PhysicalQuantity):
physicalQuantity_to_generate = PhysicalQuantity(Slot_data_label_to_generate)
if physicalQuantity_to_generate.op is None:
raise ConfigurationException('Sorry, PhysicalQuantity "{}" has no operation defined, cannot aggregate.'.format(physicalQuantity_to_generate))
#--------------------------------------
# Handle generators
#--------------------------------------
# Is this physicalQuantity generated by a custom generator defined inside the sensor class?
try:
# TODO: Use the 'provides' logic
Generator = getattr(self.Sensor, Slot_data_label_to_generate)
handled = True
except AttributeError:
pass
# Is this physicalQuantity generated by a standard generator?
try:
from luna.aggregators import generators
Generator = getattr(generators, Slot_data_label_to_generate)
handled = True
except AttributeError:
pass
#--------------------------------------
# Handle operations
#--------------------------------------
# Is this physicalQuantity_to_generate generated by applying the operation to another
# physicalQuantity_to_generate defined in the Points?
for Point_physicalQuantity in self.Sensor.Points_data_labels:
# Standard operation
if physicalQuantity_to_generate.name_unit == Point_physicalQuantity:
try:
from luna.aggregators import operations
Operation = getattr(operations, physicalQuantity_to_generate.op)
except AttributeError:
# TODO: add more info (i.e. sensor class etc?)
raise ConfigurationException('Sorry, I cannot find any valid operation for {} in luna.aggregators.operations'.format(physicalQuantity_to_generate.op))
operate_on = Point_physicalQuantity
handled = True
break
logger.debug('For generating %s I will use generator %s and operation %s', physicalQuantity_to_generate, Generator, Operation)
#----------------------
# Now compute
#----------------------
if not handled:
# TODO: add more info (i.e. sensor class etc?)
raise ConfigurationException('Could not handle "{}", as I did not find any way to generate it. Please check your configuration for this sensor'.format(Slot_data_label_to_generate))
# Set if streaming Operation/generator:
try:
Generator_is_streaming = Generator.is_stremaing
except AttributeError:
Generator_is_streaming = False
try:
Operation_is_streaming = Operation.is_stremaing
except AttributeError:
Operation_is_streaming = False
if Generator and not Generator_is_streaming:
logger.debug('Running generator %s on to generate %s', Generator, Slot_data_label_to_generate)
# A generator also requires access to the aggregated data, so we initialize it also here**
Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels,
values = Slot_data_values)
# Run the generator
result = Generator.generate(dataSeries = dataTimeSeries,
start_Point = start_Point,
end_Point = end_Point,
aggregated_data = Slot_physicalData)
# Ok, append the operation/generator results to the labels and values
logger.debug('Done running generator')
Slot_data_labels.append(Slot_data_label_to_generate)
Slot_data_values.append(result)
elif Operation and not Operation_is_streaming:
logger.debug('Running operation %s on %s to generate %s', Operation, operate_on, Slot_data_label_to_generate)
# Run the operation
result = Operation.compute_on_Points(dataSeries = dataTimeSeries.lazy_filter_data_label(label=operate_on),
start_Point = start_Point,
end_Point = end_Point)
# Ok, append the operation/generator results to the labels and values
logger.debug('Done running operation')
Slot_data_labels.append(Slot_data_label_to_generate)
Slot_data_values.append(result)
else:
raise ConsistencyException('No generator nor Operation?! (maybe got streaming which is not yet supported)')
#----------------------
# Build results
#----------------------
Slot_physicalData = self.Sensor.Points_type.data_type(labels = Slot_data_labels,
values = Slot_data_values)
dataTimeSlot = self.Sensor.Slots_type(start = start_Point,
end = end_Point,
data = Slot_physicalData,
span = timeSlotSpan,
coverage = Slot_coverage)
# Return results
logger.info('Done aggregating, slot: %s', dataTimeSlot)
return dataTimeSlot
