def filter_data(data, rules, verbose=False):
filtered_data = data
log("Filtering dataset...", verbose=verbose)
for rule in rules:
filtered_data = filter(
lambda item: item[rule['name']] != rule['value'], filtered_data)
result = list(filtered_data)
log("Records dropped:", len(data) - len(result), verbose=verbose)
return result
def run(self):
while True:
with open(dir_path + '/service/logs/log.txt', 'a') as fh:
try:
logger.log("Starting MQTT service...")
mqtt_service = IntegrationService()
mqtt_service.start()
except Exception as e:
logger.log(e)
raise
def run(self):
while True:
with open(dir_path + '/service/logs/scheduler_log.txt', 'a') as fh:
try:
logger.log("Starting Scheduler service...")
scheduler_service = Scheduler()
scheduler_service.start()
except Exception as e:
logger.log(e)
raise
def start(self):
try:
client = mqtt.Client()
client.tls_set(ca_certs=config.broker.cafile,
certfile=config.broker.clientCert,
keyfile=config.broker.clientKey)
client.on_connect = IntegrationService.on_connect
client.on_message = IntegrationService.on_message
#prisijungiam prie brokerio su confige esanciais parametrais
client.connect(config.broker.host, config.broker.port, 60)
client.loop_forever()
except Exception as Ex:
logger.log(Ex)
raise
def get_merged_rare_values_dict(values,
threshold,
default_key='OTHER',
verbose=False):
rel_f = count_relative_frequencies(values)
log(format_dict(rel_f, 3), verbose=verbose)
major_elements = {
key: value
for key, value in rel_f.items() if value >= threshold
}
nothing_excluded = len(major_elements) == len(rel_f)
only_one_value_excluded = len(major_elements) == len(rel_f) - 1
if nothing_excluded or only_one_value_excluded:
return rel_f
minor_f_sum = sum(value for value in rel_f.values() if value < threshold)
major_elements[default_key] = minor_f_sum
return major_elements
def pack_all_categorical_fields(data,
categorical_fields,
default_value='OTHER',
verbose=False):
log("Categorical fields' stats:", verbose=verbose)
for field in categorical_fields:
values = [item[field['name']] for item in data]
log(field['name'], verbose=verbose)
merged_values = get_merged_rare_values_dict(
values, field['frequency_threshold'], verbose=verbose)
log(format_dict(merged_values, 3), verbose=verbose)
for item in data:
replace_value(item,
field['name'],
merged_values,
default_value=default_value)
def remove_field_from_dataset(data, field_name, verbose=False):
log("Removing field from all records:", field_name, verbose=verbose)
for item in data:
remove_field(item, field_name)
def on_message(self, userdata, msg):
try:
device_mac = GetTopicData(msg.topic, 2)
logger.log('data received from ' + device_mac)
logger.log(msg.topic)
logger.log(msg.payload.decode('utf-8'))
#duomenu desifravimas
aes = AESCipher()
data = json.loads(msg.payload)
#tikrinam, ar duomenys yra uzsifruoto paketo formato
if ('iv' in data and 'data' in data):
key = aes.load_key(device_mac)
dec = aes.decrypt(enc=data, key=key)
if dec is None:
raise Exception('Nepavyko issifruoti!')
logger.log(json.loads(dec))
payload = JsonParse(dec)
else:
raise ValueError('Netinkamo formato paketas!')
#atidarom pagrindines DB sesija
session = Session()
#surandam DB iregistruota prietaisa, prietaiso savininka ir paskutinius duomenis
device = session.query(Devices).filter_by(mac=device_mac).first()
user = session.query(Users).filter_by(id=device.user_id).first()
if (GetTopicData(msg.topic, 0) != user.uuid):
logger.log(
"Vartotojo UUID neatitinka temoje pateikto UUID! Nutraukiama."
)
raise ValueError(
'Vartotojo UUID neatitinka temoje pateikto UUID! Nutraukiama.'
)
#naudotojo asmenines DB sesijas
user_session = userDB.create_user_session(user)
#sistemos naudotojo DB randame uzregistruota prietaisa
user_device = user_session.query(UserDevices).filter_by(
mac=device.mac).first()
#tikrinam, ar devaisas aktyvus
if (user_device.status != enums.DeviceState.Active.value
and user_device.status != enums.DeviceState.Deleted.value
and user_device.status != enums.DeviceState.Blocked.value):
user_device.status = enums.DeviceState.Active.value
if (user_device is not None):
#-------- SILDYTUVAS ----------------
if (user_device.device_type == enums.DeviceType.Heater.value):
#uzkraunam devaiso config irasa
#paskutiniai devaiso duomenys
last_data = user_session.query(
heater.HeaterData).filter_by(
device_id=user_device.id).order_by(
heater.HeaterData.date.desc()).first()
#suformuojam tema
system_name = "system"
topic = user.uuid + "/" + system_name + "/" + device.mac + "/control"
configuration = user_session.query(
heater.HeaterConfig).filter_by(
device_id=user_device.id,
is_active=True,
job_state=enums.ConfigJobState.Running.value
).first()
if (configuration is not None):
temp_treshold = configuration.temp_treshold
temp = Decimal(payload.data.temp)
if (temp_treshold is not None):
if (temp >= temp_treshold and last_data is not None
and last_data.temp < temp_treshold):
logger.log('Temperature treshold exceeded: ' +
str(temp))
device_controller.save_device_history(
user_session, user_device,
"Temperatūros riba (" +
str(temp_treshold) +
") viršyta pagal konfigūraciją \"" +
configuration.name + "\"! Temperatūra: " +
str(temp) + " Išjungiama programa.")
#riba virsyta, isjungiam rutinini darba
device_controller.execute_job(
user, user_device, configuration, True)
elif (temp < temp_treshold
and last_data is not None
and last_data.temp >= temp_treshold):
logger.log(
'Temperature lowered below treshold: ' +
str(temp))
device_controller.save_device_history(
user_session, user_device,
"Temperatūra " + str(temp) +
" žemesnė už nustatytą ribą (" +
str(temp_treshold) +
"). Atstatoma konfigūracijos \"" +
configuration.name + "\" programa.")
#temperatura sumazejo, bet rutininis darbas nesibaige
device_controller.execute_job(
user, user_device, configuration)
temp = Decimal(payload.data.temp)
actuator1 = Str2Bool(payload.data.act1)
actuator2 = Str2Bool(payload.data.act2)
actuator3 = Str2Bool(payload.data.act3)
#sukuriam duomenu irasa ir issaugom DB
new_data = heater.HeaterData(device_id=user_device.id,
temp=temp,
actuator1_state=actuator1,
actuator2_state=actuator2,
actuator3_state=actuator3)
user_session.add(new_data)
user_session.commit()
#---------- CIA KITU TIPU DEVAISAI ---------------
elif (user_device.device_type == enums.DeviceType.Default.value
):
#paskutiniai devaiso duomenys
last_data = user_session.query(
default_device.DefaultDeviceData
).filter_by(device_id=user_device.id).order_by(
default_device.DefaultDeviceData.date.desc()).first()
#suformuojam tema
system_name = "system"
topic = user.uuid + "/" + system_name + "/" + device.mac + "/control"
temp = Decimal(payload.data.temp)
ledState = Str2Bool(payload.data.ledState)
#sukuriam duomenu irasa ir issaugom DB
new_data = default_device.DefaultDeviceData(
device_id=user_device.id,
temp=temp,
led_state=ledState,
)
user_session.add(new_data)
user_session.commit()
else:
logger.log('Wrong device type! User UUID: ' + user.uuid +
' Device MAC: ' + device.mac)
else:
logger.log('Device have not assigned type! User UUID: ' +
user.uuid + ' Device MAC: ' + device.mac)
user_session.rollback()
session.rollback()
except Exception as Ex:
logger.log(Ex)
session.rollback()
user_session.rollback()
raise
finally:
session.close()
user_session.close()
def on_connect(self, userdata, flags, rc):
logger.log("Connected with result code " + str(rc))
self.subscribe("+/+/+/jsondata")