def get_sensor_data(self, data):
sd = SensorData.select(SensorData.value, SensorData.created_at) \
.where(SensorData.plant == data['plant']) \
.where(SensorData.sensor == data['sensor']) \
.order_by(SensorData.created_at.asc())
return sd
def __retrieve_data(self, target, source, sensor):
logger.info('retrieving data')
data = SensorData.select().where(SensorData.plant == target,
SensorData.sensor == sensor) \
.order_by(SensorData.created_at.desc())
saved = True
if data.count() < 1:
saved = False
data = SensorData.select().where(SensorData.plant == source,
SensorData.sensor == sensor) \
.order_by(SensorData.created_at.desc())
logger.debug('other plant data entries: {}'.format(data.count()))
logger.info('harvesting additional data')
if data.count() > 1000:
logger.debug('more than 1000 assets')
offset = random.randint(0, data.count() - 1000)
data = data.offset(offset).limit(1000)
data = data.dicts()
prepared = list(data)
current = datetime.datetime.now()
for sample in prepared:
sample['plant'] = target
sample['created_at'] = current
del sample['id']
current -= datetime.timedelta(minutes=30)
logger.debug('amount of selected data: {}'.format(len(prepared)))
with db.atomic():
for idx in range(0, len(prepared), 100):
SensorData.insert_many(prepared[idx:idx + 100]).execute()
data = SensorData.select().where(SensorData.plant == target,
SensorData.sensor == sensor) \
.order_by(SensorData.created_at.desc())
saved = True
return data, saved
def run(self):
result = VariousTools.offline_check('pump', hardware=False)
if result:
status = SensorStatus.get(SensorStatus.sensor == self.sensor,
SensorStatus.plant == self.plant)
optimum = SensorSatisfactionLevel.get(
SensorSatisfactionValue.label == 'optimum')
satisfaction = SensorSatisfactionValue.get(
SensorSatisfactionValue.sensor == self.sensor,
SensorSatisfactionValue.plant == self.plant,
SensorSatisfactionValue.level == optimum)
latest = SensorData.select(
SensorData.sensor == self.sensor,
SensorData.plant == self.plant).order_by(
SensorData.created_at.desc()).limit(1)[0]
if status.level.label == 'threat' and latest.value < satisfaction.min_value:
GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)
GPIO.setup(17, GPIO.OUT)
GPIO.setup(27, GPIO.OUT)
GPIO.setup(22, GPIO.OUT)
GPIO.output(27, True)
GPIO.output(22, False)
p = GPIO.PWM(17, 500)
p.start(0)
counter = 0
dc = 0
limited = False
while dc <= 100 and not limited:
dc += 5
print(dc)
if dc > 100 and not limited:
dc = 100
else:
p.ChangeDutyCycle(dc)
counter += 1
if counter % 2 == 0:
limited = self.check(samples=3)
time.sleep(.1)
for dc in range(dc, -1, -5):
p.ChangeDutyCycle(dc)
time.sleep(0.1)
p.stop()
GPIO.output(27, False)
GPIO.output(22, False)
def get_second_last_entry(self, current, plant):
try:
previous = SensorData.select() \
.where(SensorData.sensor == current.sensor) \
.where(SensorData.plant == plant) \
.order_by(SensorData.created_at.desc()) \
.limit(2)[1]
except:
return None
return previous
def get_plant_count(p_uuid, sensor):
plant = Plant.get(Plant.uuid == p_uuid)
sensor = Sensor.get(Sensor.name == sensor)
if plant.role != 'master' and sensor.name not in slave_supported:
plant = Plant.get(Plant.uuid == UUID(plant.role))
sensor_data_set = SensorData.select() \
.where(SensorData.plant == plant) \
.where(SensorData.sensor == sensor)
return json.dumps({'count': sensor_data_set.count()})
def get_plant_sensor_data(p_uuid, sensor):
plant = Plant.get(Plant.uuid == p_uuid)
sensor = Sensor.get(Sensor.name == sensor)
if plant.role != 'master' and sensor.name not in slave_supported:
plant = Plant.get(Plant.uuid == UUID(plant.role))
sensor_data_set = SensorData.select(SensorData.value, fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \
.where(SensorData.plant == plant) \
.where(SensorData.sensor == sensor) \
.order_by(SensorData.created_at.asc())
content = list(sensor_data_set)
return json.dumps(content)
def get_plant_data_selective(p_uuid, sensor, start, stop):
plant = Plant.get(uuid=p_uuid)
sensor = Sensor.get(name=sensor)
if plant.role != 'master' and sensor.name not in slave_supported:
plant = Plant.get(uuid=plant.role)
dataset = SensorData.select(SensorData.value, fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \
.where(SensorData.plant == plant) \
.where(SensorData.sensor == sensor) \
.order_by(SensorData.created_at.desc()) \
.offset(start) \
.limit(stop - start) \
.dicts()
dataset = list(dataset)
return json.dumps(dataset)
def delete_non_persistant_overflow(self, sensor, plant):
deleted = 0
non_persistant = SensorData.select() \
.where(SensorData.sensor == sensor) \
.where(SensorData.plant == plant) \
.where(SensorData.persistant == False) \
.order_by(SensorData.created_at.asc())
# .order_by(SensorData.created_at.desc())
overflow = non_persistant.count() - plant.persistant_hold
overflow = int(overflow)
if overflow > 0:
for dataset in non_persistant[:overflow]:
deleted += dataset.delete_instance()
return deleted
def persistant_evaluation(self, plant, sensor):
dataset = SensorData.select(SensorData.created_at) \
.where(SensorData.sensor == sensor) \
.where(SensorData.plant == plant) \
.where(SensorData.persistant == True) \
.order_by(SensorData.created_at.desc()) \
.limit(1) \
.dicts()
if dataset.count() == 0:
return True
else:
dataset = list(dataset)[0]
if datetime.datetime.now(
) - dataset['created_at'] >= datetime.timedelta(minutes=29):
return True
else:
return False
def get_latest_dataset(p_uuid, s_uuid):
plant = Plant.get(Plant.uuid == p_uuid)
sensor = Sensor.get(Sensor.name == s_uuid)
if plant.role != 'master' and sensor.name not in slave_supported:
plant = Plant.get(Plant.uuid == UUID(plant.role))
sd = SensorData.select(SensorData.value,
SensorData.persistant,
fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \
.where(SensorData.plant == plant) \
.where(SensorData.sensor == sensor) \
.order_by(SensorData.created_at.desc()) \
.limit(1) \
.dicts()
selected = list(sd)[0]
return json.dumps(selected)
def get_plant_current_sensor_data(p_uuid, sensor):
plant = Plant.get(Plant.uuid == p_uuid)
sensor = Sensor.get(Sensor.name == sensor)
if plant.role != 'master' and sensor.name not in slave_supported:
plant = Plant.get(Plant.uuid == UUID(plant.role))
latest = SensorData.select() \
.where(SensorData.plant == plant) \
.where(SensorData.sensor == sensor) \
.order_by(SensorData.created_at.desc())[0]
latest = model_to_dict(latest)
del latest['id']
del latest['plant']
del latest['sensor']
return json.dumps(latest, default=json_util.default)
def get_plant_sensor_data_after(p_uuid, sensor, until):
sensor = Sensor.get(Sensor.name == sensor)
plant = Plant.get(Plant.uuid == p_uuid)
if plant.role != 'master' and sensor.name not in slave_supported:
plant = Plant.get(Plant.uuid == UUID(plant.role))
date_time = datetime.datetime.fromtimestamp(until + 1)
sensor_data_set = SensorData.select(SensorData.value,
fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \
.where(SensorData.plant == plant) \
.where(SensorData.sensor == sensor) \
.where(SensorData.created_at > date_time) \
.order_by(SensorData.created_at.asc()) \
.dicts()
sensor_data_set = list(sensor_data_set)
return json.dumps(sensor_data_set)
def get_sensor_data_high_low(plant, sensor, configuration, target=None):
mode = SensorData.created_at.asc(
) if configuration is False else SensorData.created_at.desc()
dataset = SensorData.select(SensorData.value.alias('v'),
fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('t')) \
.where(SensorData.plant == plant) \
.where(SensorData.sensor == sensor) \
.order_by(mode)
if target is not None:
dataset = dataset.where(SensorData.created_at >= target)
dataset = dataset.limit(1).dicts()
if dataset.count() == 0:
return None
data = list(dataset)[0]
return data
def run(self, target, sensor, source=None):
if source is None:
count = 0
for plant in Plant.select():
current = SensorData.select().where(SensorData.plant == plant,
SensorData.sensor == sensor) \
.count()
if count < current:
count = current
source = plant
predicted = SensorDataPrediction.select().where(SensorDataPrediction.plant == target,
SensorDataPrediction.sensor == sensor)
logger.info('insertations left: {}'.format(predicted.count()))
if predicted.count() == 0:
data, saved = self.__retrieve_data(target, source, sensor)
data = data.order_by(SensorData.created_at.asc())
forecast = SensorDataForecast()
information = {'plant': target, 'sensor': sensor}
information['prediction'] = forecast.predict(information, data)
forecast.insert_database(information)
predicted = SensorDataPrediction.select().where(SensorDataPrediction.plant == target,
SensorDataPrediction.sensor == sensor) \
.order_by(SensorDataPrediction.created_at.asc()) \
.limit(1)
for sample in predicted:
if sample.time <= datetime.datetime.now():
released = True
ToolChainSensor().insert_data({'sensor': sensor, 'plant': target, 'value': sample.value}, prediction=False)
logger.info('inserting')
# insert hardware? -- test if DUMMYPLANT hardware?
sample.delete_instance()
else:
released = False
logger.info('too early to insert')
return released
def get(self):
sensors = self.data['sensor']
plant = Plant.select().where(Plant.localhost == True)[0]
for sensor in sensors:
sensors[sensor]['raw'] = Sensor.get(Sensor.name == sensor)
sensors[sensor]['name'] = sensors[sensor]['raw'].name
unit = sensors[sensor]['raw'].unit
sensors[sensor]['unit'] = re.sub(r'(?!%)\W+', '', unit)[0]
order = SensorData.created_at.desc()
r_sen = sensors[sensor]['raw']
sensors[sensor]['value'] = SensorData.select()\
.where(SensorData.plant == plant)\
.where(SensorData.sensor == r_sen)\
.order_by(order)[0].value
self.data['sensor'] = sensors
return self
from models.sensor import SensorData
import datetime
count = SensorData.select().count()
i = 0
j = 0
for data in SensorData.select():
if isinstance(data.created_at, str):
try:
data.created_at = data.created_at.replace('+00:00', '')
data.created_at = datetime.datetime.strptime(
data.created_at, '%Y-%m-%d %H:%M:%S')
except:
data.created_at = datetime.datetime.strptime(
data.created_at, "%Y-%m-%d %H:%M:%S.%f")
i += 1
data.save()
j += 1
if j % 1000 == 0:
print('done: {}%'.format(round(j / count * 100, 2)))
if i % 1000 == 0:
print('processed: {}'.format(i))
print('-------------------')
print('done: {}%'.format(round(j / count * 100, 2)))
print('processed: {}'.format(i))
def insert_data(self, data, mesh=True, prediction=True):
""" dict of data:
'sensor': object of sensor
'value': value - float
'plant': current selected plant
"""
start = datetime.datetime.now()
current_entries = SensorData.select()\
.where(SensorData.sensor == data['sensor'])\
.count()
persistant = False
data['value'] = round(data['value'], 2)
sensor_db = SensorData()
sensor_db.value = data['value']
sensor_db.plant = data['plant']
sensor_db.sensor = data['sensor']
sensor_db.persistant = False
sensor_db.save()
last_entry = self.get_second_last_entry(sensor_db, data['plant'])
last_value = last_entry.value if last_entry is not None else data[
'value']
offset = abs(data['value'] - last_value)
if offset >= data['sensor'].persistant_offset:
persistant = True
elif current_entries > 6:
persistant = self.persistant_evaluation(data['plant'],
data['sensor'])
sensor_db.persistant = persistant
sensor_db.save()
self.delete_non_persistant_overflow(data['sensor'], data['plant'])
logger.debug('{} - {} persistant: {}'.format(data['plant'].name,
data['sensor'].name,
persistant))
if persistant:
data['satisfaction'] = self.modify_sensor_status(data, mesh)
self.mail_evaluation(data)
if prediction:
SensorDataForecast().run(data)
if mesh:
from mesh_network.dedicated import MeshDedicatedDispatch
MeshDedicatedDispatch().new_data(data['sensor'],
plant=data['plant'])
if data['plant'].localhost:
print('slave data')
slaves = Plant.select().where(
Plant.role == str(data['plant'].uuid))
slaves = list(slaves)
for slave in slaves:
print('slaved')
MeshDedicatedDispatch().slave_data(
slave, data['sensor'])
logger.debug('time elapsed: {}'.format(datetime.datetime.now() -
start))
return persistant
from sensor_scripts.extensions.mailer import PlantMailer
# msg = MessagePreset()
# msg.name = 'en'
# msg.message = '[date] | [time (12h)]\n[plant] - [sensor]\n[current][unit]\n[warning_min] - [warning_max]'
# msg.default = True
# msg.save()
# wizard = Person.get(Person.wizard == True)
# wizard.preset = MessagePreset.get(MessagePreset.name == 'en')
# wizard.save()
msg = MessagePreset.get(MessagePreset.name == 'en')
msg.message = '[date] | [time (12h)]\n[plant] - [sensor]\nvalue: [current][unit]\nwarning: [warning_min] - [warning_max]'
msg.save()
messages = SensorData.select().order_by(
SensorData.created_at.desc()).offset(10).limit(10)
mailer = PlantMailer()
mailer.format_messages(messages)
# t = SensorSatisfactionLevel.get(SensorSatisfactionLevel.label == 'threat')
# for data in SensorData.select().order_by(SensorData.created_at.desc()).limit(10):
# message = SensorDangerMessage()
# message.plant = data.plant
# message.sensor = data.sensor
# message.level = t
# message.created_at = data.created_at
# message.message = '---'
# message.value = data.value
def plant_sensor(p_uuid, s_uuid):
# GET: select: latest, prediction, data, current, range, extreme, message
# GET: max: Boolean (extreme)
# GET: ever: Boolean (extreme)
# GET: backlog: Boolean (extreme)
# GET: start: Integer (data)
# GET: stop: Integer (data)
# GET: count: Boolean (data)
plant = Plant.get(uuid=p_uuid)
try:
sensor = Sensor.get(uuid=s_uuid)
except Exception:
sensor = Sensor.get(name=s_uuid)
if plant.role != 'master' and sensor.name not in slave_supported:
plant = Plant.get(uuid=plant.role)
data, code = get_data(required=PLANT_SENSOR_GET, restrictive=True)
if code == 400:
return data_formatting(400)
selector = data['select']
maximum = data['max']
backlog = data['backlog']
start = data['start']
stop = data['stop']
ever = data['ever']
collection = {}
for selected in selector:
if selected == 'timespan':
start = datetime.datetime.min if start <= 0 else datetime.datetime.fromtimestamp(
start)
stop = datetime.datetime.max if stop >= 253402297200 else datetime.datetime.fromtimestamp(
stop)
if selected in [
'latest', 'current', 'extreme', 'data', 'count', 'timespan'
]:
data = SensorData.select(SensorData.value,
SensorData.persistant,
fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \
.where(SensorData.plant == plant) \
.where(SensorData.sensor == sensor) \
.order_by(SensorData.created_at.asc()) \
.dicts()
elif selected in ['prediction']:
data = SensorDataPrediction.select(SensorDataPrediction.value,
fn.CAST(Clause(fn.strftime('%s', SensorDataPrediction.time), SQL('AS INT'))).alias('timestamp')) \
.where(SensorDataPrediction.plant == plant) \
.where(SensorDataPrediction.sensor == sensor) \
.order_by(SensorDataPrediction.created_at.asc()) \
.dicts()
elif selected in ['range']:
data = SensorSatisfactionValue.select() \
.where(SensorSatisfactionValue.plant == plant) \
.where(SensorSatisfactionValue.sensor == sensor)
if selected in ['latest', 'current']:
data = data.order_by(SensorData.created_at.desc()).limit(1)
output = list(data)[0]
elif selected == 'prediction':
output = list(data)
elif selected == 'range':
output = {}
for spectrum in data:
output[spectrum.level.name_color] = {
'max': spectrum.max_value,
'min': spectrum.min_value
}
if spectrum.level.name_color == 'red':
output[spectrum.level.name_color] = {
'max': sensor.max_value,
'min': sensor.min_value
}
elif selected == 'extreme':
target = datetime.datetime.combine(datetime.date.today(),
datetime.datetime.min.time())
if backlog and not ever:
output = None
while output is None:
output = get_sensor_data_high_low(plant, sensor, maximum,
target)
target = target - datetime.timedelta(days=1)
else:
if ever:
target = None
output = get_sensor_data_high_low(plant, sensor, maximum,
target)
elif selected == 'count':
output = data.count()
elif selected == 'timespan':
data = data.where(SensorData.created_at > start,
SensorData.created_at < stop)
output = list(data)
else:
data = data.order_by(
SensorData.created_at.desc()).offset(start).limit(stop - start)
output = list(data)
if len(selector) > 1:
collection[selected] = output
if len(collection.keys()) != 0:
output = collection
return data_formatting(data=output)