def get_sensor_data(self, data): sd = SensorData.select(SensorData.value, SensorData.created_at) \ .where(SensorData.plant == data['plant']) \ .where(SensorData.sensor == data['sensor']) \ .order_by(SensorData.created_at.asc()) return sd
def __retrieve_data(self, target, source, sensor): logger.info('retrieving data') data = SensorData.select().where(SensorData.plant == target, SensorData.sensor == sensor) \ .order_by(SensorData.created_at.desc()) saved = True if data.count() < 1: saved = False data = SensorData.select().where(SensorData.plant == source, SensorData.sensor == sensor) \ .order_by(SensorData.created_at.desc()) logger.debug('other plant data entries: {}'.format(data.count())) logger.info('harvesting additional data') if data.count() > 1000: logger.debug('more than 1000 assets') offset = random.randint(0, data.count() - 1000) data = data.offset(offset).limit(1000) data = data.dicts() prepared = list(data) current = datetime.datetime.now() for sample in prepared: sample['plant'] = target sample['created_at'] = current del sample['id'] current -= datetime.timedelta(minutes=30) logger.debug('amount of selected data: {}'.format(len(prepared))) with db.atomic(): for idx in range(0, len(prepared), 100): SensorData.insert_many(prepared[idx:idx + 100]).execute() data = SensorData.select().where(SensorData.plant == target, SensorData.sensor == sensor) \ .order_by(SensorData.created_at.desc()) saved = True return data, saved
def run(self): result = VariousTools.offline_check('pump', hardware=False) if result: status = SensorStatus.get(SensorStatus.sensor == self.sensor, SensorStatus.plant == self.plant) optimum = SensorSatisfactionLevel.get( SensorSatisfactionValue.label == 'optimum') satisfaction = SensorSatisfactionValue.get( SensorSatisfactionValue.sensor == self.sensor, SensorSatisfactionValue.plant == self.plant, SensorSatisfactionValue.level == optimum) latest = SensorData.select( SensorData.sensor == self.sensor, SensorData.plant == self.plant).order_by( SensorData.created_at.desc()).limit(1)[0] if status.level.label == 'threat' and latest.value < satisfaction.min_value: GPIO.setmode(GPIO.BCM) GPIO.setwarnings(False) GPIO.setup(17, GPIO.OUT) GPIO.setup(27, GPIO.OUT) GPIO.setup(22, GPIO.OUT) GPIO.output(27, True) GPIO.output(22, False) p = GPIO.PWM(17, 500) p.start(0) counter = 0 dc = 0 limited = False while dc <= 100 and not limited: dc += 5 print(dc) if dc > 100 and not limited: dc = 100 else: p.ChangeDutyCycle(dc) counter += 1 if counter % 2 == 0: limited = self.check(samples=3) time.sleep(.1) for dc in range(dc, -1, -5): p.ChangeDutyCycle(dc) time.sleep(0.1) p.stop() GPIO.output(27, False) GPIO.output(22, False)
def get_second_last_entry(self, current, plant): try: previous = SensorData.select() \ .where(SensorData.sensor == current.sensor) \ .where(SensorData.plant == plant) \ .order_by(SensorData.created_at.desc()) \ .limit(2)[1] except: return None return previous
def get_plant_count(p_uuid, sensor): plant = Plant.get(Plant.uuid == p_uuid) sensor = Sensor.get(Sensor.name == sensor) if plant.role != 'master' and sensor.name not in slave_supported: plant = Plant.get(Plant.uuid == UUID(plant.role)) sensor_data_set = SensorData.select() \ .where(SensorData.plant == plant) \ .where(SensorData.sensor == sensor) return json.dumps({'count': sensor_data_set.count()})
def get_plant_sensor_data(p_uuid, sensor): plant = Plant.get(Plant.uuid == p_uuid) sensor = Sensor.get(Sensor.name == sensor) if plant.role != 'master' and sensor.name not in slave_supported: plant = Plant.get(Plant.uuid == UUID(plant.role)) sensor_data_set = SensorData.select(SensorData.value, fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \ .where(SensorData.plant == plant) \ .where(SensorData.sensor == sensor) \ .order_by(SensorData.created_at.asc()) content = list(sensor_data_set) return json.dumps(content)
def get_plant_data_selective(p_uuid, sensor, start, stop): plant = Plant.get(uuid=p_uuid) sensor = Sensor.get(name=sensor) if plant.role != 'master' and sensor.name not in slave_supported: plant = Plant.get(uuid=plant.role) dataset = SensorData.select(SensorData.value, fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \ .where(SensorData.plant == plant) \ .where(SensorData.sensor == sensor) \ .order_by(SensorData.created_at.desc()) \ .offset(start) \ .limit(stop - start) \ .dicts() dataset = list(dataset) return json.dumps(dataset)
def delete_non_persistant_overflow(self, sensor, plant): deleted = 0 non_persistant = SensorData.select() \ .where(SensorData.sensor == sensor) \ .where(SensorData.plant == plant) \ .where(SensorData.persistant == False) \ .order_by(SensorData.created_at.asc()) # .order_by(SensorData.created_at.desc()) overflow = non_persistant.count() - plant.persistant_hold overflow = int(overflow) if overflow > 0: for dataset in non_persistant[:overflow]: deleted += dataset.delete_instance() return deleted
def persistant_evaluation(self, plant, sensor): dataset = SensorData.select(SensorData.created_at) \ .where(SensorData.sensor == sensor) \ .where(SensorData.plant == plant) \ .where(SensorData.persistant == True) \ .order_by(SensorData.created_at.desc()) \ .limit(1) \ .dicts() if dataset.count() == 0: return True else: dataset = list(dataset)[0] if datetime.datetime.now( ) - dataset['created_at'] >= datetime.timedelta(minutes=29): return True else: return False
def get_plant_current_sensor_data(p_uuid, sensor): plant = Plant.get(Plant.uuid == p_uuid) sensor = Sensor.get(Sensor.name == sensor) if plant.role != 'master' and sensor.name not in slave_supported: plant = Plant.get(Plant.uuid == UUID(plant.role)) latest = SensorData.select() \ .where(SensorData.plant == plant) \ .where(SensorData.sensor == sensor) \ .order_by(SensorData.created_at.desc())[0] latest = model_to_dict(latest) del latest['id'] del latest['plant'] del latest['sensor'] return json.dumps(latest, default=json_util.default)
def get_plant_sensor_data_after(p_uuid, sensor, until): sensor = Sensor.get(Sensor.name == sensor) plant = Plant.get(Plant.uuid == p_uuid) if plant.role != 'master' and sensor.name not in slave_supported: plant = Plant.get(Plant.uuid == UUID(plant.role)) date_time = datetime.datetime.fromtimestamp(until + 1) sensor_data_set = SensorData.select(SensorData.value, fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \ .where(SensorData.plant == plant) \ .where(SensorData.sensor == sensor) \ .where(SensorData.created_at > date_time) \ .order_by(SensorData.created_at.asc()) \ .dicts() sensor_data_set = list(sensor_data_set) return json.dumps(sensor_data_set)
def get_latest_dataset(p_uuid, s_uuid): plant = Plant.get(Plant.uuid == p_uuid) sensor = Sensor.get(Sensor.name == s_uuid) if plant.role != 'master' and sensor.name not in slave_supported: plant = Plant.get(Plant.uuid == UUID(plant.role)) sd = SensorData.select(SensorData.value, SensorData.persistant, fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \ .where(SensorData.plant == plant) \ .where(SensorData.sensor == sensor) \ .order_by(SensorData.created_at.desc()) \ .limit(1) \ .dicts() selected = list(sd)[0] return json.dumps(selected)
def get_sensor_data_high_low(plant, sensor, configuration, target=None): mode = SensorData.created_at.asc( ) if configuration is False else SensorData.created_at.desc() dataset = SensorData.select(SensorData.value.alias('v'), fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('t')) \ .where(SensorData.plant == plant) \ .where(SensorData.sensor == sensor) \ .order_by(mode) if target is not None: dataset = dataset.where(SensorData.created_at >= target) dataset = dataset.limit(1).dicts() if dataset.count() == 0: return None data = list(dataset)[0] return data
def run(self, target, sensor, source=None): if source is None: count = 0 for plant in Plant.select(): current = SensorData.select().where(SensorData.plant == plant, SensorData.sensor == sensor) \ .count() if count < current: count = current source = plant predicted = SensorDataPrediction.select().where(SensorDataPrediction.plant == target, SensorDataPrediction.sensor == sensor) logger.info('insertations left: {}'.format(predicted.count())) if predicted.count() == 0: data, saved = self.__retrieve_data(target, source, sensor) data = data.order_by(SensorData.created_at.asc()) forecast = SensorDataForecast() information = {'plant': target, 'sensor': sensor} information['prediction'] = forecast.predict(information, data) forecast.insert_database(information) predicted = SensorDataPrediction.select().where(SensorDataPrediction.plant == target, SensorDataPrediction.sensor == sensor) \ .order_by(SensorDataPrediction.created_at.asc()) \ .limit(1) for sample in predicted: if sample.time <= datetime.datetime.now(): released = True ToolChainSensor().insert_data({'sensor': sensor, 'plant': target, 'value': sample.value}, prediction=False) logger.info('inserting') # insert hardware? -- test if DUMMYPLANT hardware? sample.delete_instance() else: released = False logger.info('too early to insert') return released
def get(self): sensors = self.data['sensor'] plant = Plant.select().where(Plant.localhost == True)[0] for sensor in sensors: sensors[sensor]['raw'] = Sensor.get(Sensor.name == sensor) sensors[sensor]['name'] = sensors[sensor]['raw'].name unit = sensors[sensor]['raw'].unit sensors[sensor]['unit'] = re.sub(r'(?!%)\W+', '', unit)[0] order = SensorData.created_at.desc() r_sen = sensors[sensor]['raw'] sensors[sensor]['value'] = SensorData.select()\ .where(SensorData.plant == plant)\ .where(SensorData.sensor == r_sen)\ .order_by(order)[0].value self.data['sensor'] = sensors return self
def plant_sensor(p_uuid, s_uuid): # GET: select: latest, prediction, data, current, range, extreme, message # GET: max: Boolean (extreme) # GET: ever: Boolean (extreme) # GET: backlog: Boolean (extreme) # GET: start: Integer (data) # GET: stop: Integer (data) # GET: count: Boolean (data) plant = Plant.get(uuid=p_uuid) try: sensor = Sensor.get(uuid=s_uuid) except Exception: sensor = Sensor.get(name=s_uuid) if plant.role != 'master' and sensor.name not in slave_supported: plant = Plant.get(uuid=plant.role) data, code = get_data(required=PLANT_SENSOR_GET, restrictive=True) if code == 400: return data_formatting(400) selector = data['select'] maximum = data['max'] backlog = data['backlog'] start = data['start'] stop = data['stop'] ever = data['ever'] collection = {} for selected in selector: if selected == 'timespan': start = datetime.datetime.min if start <= 0 else datetime.datetime.fromtimestamp( start) stop = datetime.datetime.max if stop >= 253402297200 else datetime.datetime.fromtimestamp( stop) if selected in [ 'latest', 'current', 'extreme', 'data', 'count', 'timespan' ]: data = SensorData.select(SensorData.value, SensorData.persistant, fn.CAST(Clause(fn.strftime('%s', SensorData.created_at), SQL('AS INT'))).alias('timestamp')) \ .where(SensorData.plant == plant) \ .where(SensorData.sensor == sensor) \ .order_by(SensorData.created_at.asc()) \ .dicts() elif selected in ['prediction']: data = SensorDataPrediction.select(SensorDataPrediction.value, fn.CAST(Clause(fn.strftime('%s', SensorDataPrediction.time), SQL('AS INT'))).alias('timestamp')) \ .where(SensorDataPrediction.plant == plant) \ .where(SensorDataPrediction.sensor == sensor) \ .order_by(SensorDataPrediction.created_at.asc()) \ .dicts() elif selected in ['range']: data = SensorSatisfactionValue.select() \ .where(SensorSatisfactionValue.plant == plant) \ .where(SensorSatisfactionValue.sensor == sensor) if selected in ['latest', 'current']: data = data.order_by(SensorData.created_at.desc()).limit(1) output = list(data)[0] elif selected == 'prediction': output = list(data) elif selected == 'range': output = {} for spectrum in data: output[spectrum.level.name_color] = { 'max': spectrum.max_value, 'min': spectrum.min_value } if spectrum.level.name_color == 'red': output[spectrum.level.name_color] = { 'max': sensor.max_value, 'min': sensor.min_value } elif selected == 'extreme': target = datetime.datetime.combine(datetime.date.today(), datetime.datetime.min.time()) if backlog and not ever: output = None while output is None: output = get_sensor_data_high_low(plant, sensor, maximum, target) target = target - datetime.timedelta(days=1) else: if ever: target = None output = get_sensor_data_high_low(plant, sensor, maximum, target) elif selected == 'count': output = data.count() elif selected == 'timespan': data = data.where(SensorData.created_at > start, SensorData.created_at < stop) output = list(data) else: data = data.order_by( SensorData.created_at.desc()).offset(start).limit(stop - start) output = list(data) if len(selector) > 1: collection[selected] = output if len(collection.keys()) != 0: output = collection return data_formatting(data=output)
from models.sensor import SensorData import datetime count = SensorData.select().count() i = 0 j = 0 for data in SensorData.select(): if isinstance(data.created_at, str): try: data.created_at = data.created_at.replace('+00:00', '') data.created_at = datetime.datetime.strptime( data.created_at, '%Y-%m-%d %H:%M:%S') except: data.created_at = datetime.datetime.strptime( data.created_at, "%Y-%m-%d %H:%M:%S.%f") i += 1 data.save() j += 1 if j % 1000 == 0: print('done: {}%'.format(round(j / count * 100, 2))) if i % 1000 == 0: print('processed: {}'.format(i)) print('-------------------') print('done: {}%'.format(round(j / count * 100, 2))) print('processed: {}'.format(i))
from sensor_scripts.extensions.mailer import PlantMailer # msg = MessagePreset() # msg.name = 'en' # msg.message = '[date] | [time (12h)]\n[plant] - [sensor]\n[current][unit]\n[warning_min] - [warning_max]' # msg.default = True # msg.save() # wizard = Person.get(Person.wizard == True) # wizard.preset = MessagePreset.get(MessagePreset.name == 'en') # wizard.save() msg = MessagePreset.get(MessagePreset.name == 'en') msg.message = '[date] | [time (12h)]\n[plant] - [sensor]\nvalue: [current][unit]\nwarning: [warning_min] - [warning_max]' msg.save() messages = SensorData.select().order_by( SensorData.created_at.desc()).offset(10).limit(10) mailer = PlantMailer() mailer.format_messages(messages) # t = SensorSatisfactionLevel.get(SensorSatisfactionLevel.label == 'threat') # for data in SensorData.select().order_by(SensorData.created_at.desc()).limit(10): # message = SensorDangerMessage() # message.plant = data.plant # message.sensor = data.sensor # message.level = t # message.created_at = data.created_at # message.message = '---' # message.value = data.value
from models.plant import Plant from models.sensor import SensorData, SensorDataPrediction target = Plant.get(Plant.name == 'Thomas') target.name = 'thomas' target.save() SensorData.delete().where(SensorData.plant == target).execute() SensorDataPrediction.delete().where( SensorDataPrediction.plant == target).execute()
def insert_data(self, data, mesh=True, prediction=True): """ dict of data: 'sensor': object of sensor 'value': value - float 'plant': current selected plant """ start = datetime.datetime.now() current_entries = SensorData.select()\ .where(SensorData.sensor == data['sensor'])\ .count() persistant = False data['value'] = round(data['value'], 2) sensor_db = SensorData() sensor_db.value = data['value'] sensor_db.plant = data['plant'] sensor_db.sensor = data['sensor'] sensor_db.persistant = False sensor_db.save() last_entry = self.get_second_last_entry(sensor_db, data['plant']) last_value = last_entry.value if last_entry is not None else data[ 'value'] offset = abs(data['value'] - last_value) if offset >= data['sensor'].persistant_offset: persistant = True elif current_entries > 6: persistant = self.persistant_evaluation(data['plant'], data['sensor']) sensor_db.persistant = persistant sensor_db.save() self.delete_non_persistant_overflow(data['sensor'], data['plant']) logger.debug('{} - {} persistant: {}'.format(data['plant'].name, data['sensor'].name, persistant)) if persistant: data['satisfaction'] = self.modify_sensor_status(data, mesh) self.mail_evaluation(data) if prediction: SensorDataForecast().run(data) if mesh: from mesh_network.dedicated import MeshDedicatedDispatch MeshDedicatedDispatch().new_data(data['sensor'], plant=data['plant']) if data['plant'].localhost: print('slave data') slaves = Plant.select().where( Plant.role == str(data['plant'].uuid)) slaves = list(slaves) for slave in slaves: print('slaved') MeshDedicatedDispatch().slave_data( slave, data['sensor']) logger.debug('time elapsed: {}'.format(datetime.datetime.now() - start)) return persistant
i = 0 for asset in db.SensorData.find({ "_id": { '$gt': ObjectId("57e954100000000000000000") } }).sort("_id", pymongo.ASCENDING): if asset['p'] == 'm': if asset['s'] == 't': sensor = temperature elif asset['s'] == 'h': sensor = humidity elif asset['s'] == 'l': sensor = light else: sensor = moisture imported = SensorData() imported.value = asset['v'] imported.sensor = sensor imported.plant = plant imported.persistant = True imported.created_at = datetime.datetime.fromtimestamp( asset['_id'].generation_time.timestamp()) imported.save() i += 1 if i % 100 == 0: print(i) print(i)