class Monitor_Event_Queues():
def __init__(self, rabbitmq_remote_connections, query_configuration,
chain_flow):
self.rc = rabbitmq_remote_connections
self.qc = query_configuration
self.cf = chain_flow
def process_card(self, card_dict, event_data):
card_node = card_dict["card_node"]
label = card_dict["card_action"]["label"]
#print "event_data",event_data["event"]
#print "card action",card_dict["card_action"].keys()
if label == "fromevent":
label = event_data["status"].lower()
#print "---------->",label,event_data
diag_text = "New Event: " + event_data[
"event"] + " Data: " + json.dumps(event_data)
print "diag_text ", diag_text, label
try:
temp = json.loads(card_node.properties["new_commit"])
if type(temp) is not list:
print "is not a list"
temp = []
except:
print "exception"
temp = []
temp.append(diag_text)
print "temp", temp
print "len", len(temp)
card_node.properties["new_commit"] = json.dumps(temp)
card_node.properties["label"] = label
card_node.push()
def get_cards(self, controller, cards_actions):
return_value = {}
for i in cards_actions.keys():
card_name = cards_actions[i]["card"]
card = self.qc.match_relation_property_specific(
"CONTROLLER", "namespace", controller.properties["namespace"],
"DIAGNOSTIC_CARD", "name", card_name)
if len(card) > 0:
#print "card_name",card_name
return_value[i] = {
"card_node": card[0],
"card_action": cards_actions[i]
}
return return_value
def monitor_event_queue_cf(self, *args):
self.monitor_event_queue(50)
def monitor_event_queue(self, search_depth, *args):
ref_time = time.time()
self.rc = Rabbitmq_Remote_Connections()
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
vhost = i.properties["vhost"]
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control(
i.properties["vhost"])
queue_list = self.qc.match_relation_property(
"CONTROLLER", "namespace", i.properties["namespace"],
"EVENT_QUEUE")
for j in queue_list:
cards_actions = json.loads(j.properties["events"])
cards = self.get_cards(i, cards_actions)
redis_queue = j.properties["name"]
if j.properties["timestamp"] + 48 * 3600 < time.time():
timestamp = time.time() - 48 * 3600
else:
timestamp = j.properties["timestamp"]
timestamp_max = timestamp
queue_length = int(
station_control.redis_llen([redis_queue])[1][0]["data"])
search_interval = range(0, search_depth)
search_interval.reverse()
for k in search_interval:
data = station_control.redis_lindex([{
"key": redis_queue,
"index": k
}])
if data[0] == True:
data_base64 = data[1][0]["data"]
data_json = base64.b64decode(data_base64)
data = json.loads(data_json)
if data["time"] > timestamp_max:
timestamp_max = data["time"]
if timestamp < data["time"]:
#print k,time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(data["time"])), data["status"], data["event"]
event = data["event"]
if cards.has_key(event):
self.process_card(cards[event], data)
else:
pass
#print k,"no card match"
print "time_stamps", j.properties["timestamp"], timestamp_max
j.properties["timestamp"] = timestamp_max
j.push()
class Analyize_Remote_Connectivity:
def __init__(self, rabbit_interface, query_control):
self.rc = rabbit_interface
self.qc = query_control
def convert_to_int(self, data):
return_value = {}
for i in data.keys():
return_value[i] = int(data[i])
return return_value
def transform_data(self, data):
data = self.convert_to_int(data)
error_rate = float(data["failures"]) / float(data["counts"])
message_loss = data["total_failures"]
return error_rate, message_loss
def analyize_data(self, *args):
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control(
i.properties["vhost"])
udp_server_list = self.qc.match_relation_property(
"CONTROLLER", "name", i.properties["name"], "UDP_IO_SERVER")
for j in udp_server_list:
redis_key = j.properties["redis_key"]
data = station_control.redis_hget_all([{
"hash": redis_key
}])[1][0]["data"]
remote_list = self.qc.match_relation_property(
"UDP_IO_SERVER", "namespace", j.properties["namespace"],
"REMOTE")
for k in remote_list:
modbus_address = str(k.properties["modbus_address"])
remote_data = data[modbus_address]
error_rate, message_loss = self.transform_data(
json.loads(remote_data))
card_dict = json.loads(k.properties["card_dict"])
card = self.qc.match_relation_property_specific(
"CONTROLLER", "namespace", i.properties["namespace"],
"DIAGNOSTIC_CARD", "name",
card_dict["connectivity"])[0]
text = "Controller: " + i.properties[
"name"] + " Remote: " + k.properties["name"] + "\n"
if (error_rate < .03) and (message_loss == 0):
text = text + "Error Rate: " + str(
error_rate) + " Message Loss: " + str(message_loss)
color = "green"
elif message_loss == 0:
text = text + "**Error Rate: " + str(
error_rate) + " Message Loss: " + str(
message_loss) + " ERROR **"
color = "yellow"
else:
text = text + "**Error Rate: " + str(
error_rate) + " Message Loss: " + str(
message_loss) + " ERROR **"
color = "red"
try:
temp = json.loads(card.properties["new_commit"])
if type(temp) is not list:
temp = []
except:
temp = []
temp.append(text)
card.properties["new_commit"] = json.dumps(temp)
card.properties["label"] = color
card.push()
class Analyize_Controller_Parameters():
def __init__(self, rabbitmq_remote_connections, query_configuration):
self.rc = rabbitmq_remote_connections
self.qc = query_configuration
def process_clean_filter(self, title, value):
value = float(value)
ref = time.time()
hour_sec = 3600
if value > ref - 30 * hour_sec:
color = "green"
text = "Title: " + str(title) + " Value: " + str(
(ref - value) / hour_sec) + " hours earlier "
elif value > ref - 48 * hour_sec:
color = "yellow"
text = "**Error Title: " + str(title) + " Value: " + str(
(ref - value) / hour_sec) + " hours earlier **"
else:
color = "red"
text = "**Error Title: " + str(title) + " Value: " + str(
(ref - value) / hour_sec) + " hours earlier **"
return color, text
def update_properties(self, card, limits, title, value):
if hasattr(limits, '__call__') == True:
color, text = limits(title, value)
else:
value = float(value)
if value < limits["yellow"]:
color = "green"
text = "Title: " + str(title) + " Value: " + str(value)
elif value < limits["red"]:
color = "yellow"
text = "**Error Title: " + str(title) + " Value: " + str(
value) + " **"
else:
color = "red"
text = "**Error Title: " + str(title) + " Value: " + str(
value) + " **"
try:
temp = json.loads(card.properties["new_commit"])
if type(temp) is not list:
temp = []
except:
temp = []
#print "title",title,value,color,text
temp.append(text)
card.properties["new_commit"] = json.dumps(temp)
card.properties["label"] = color
card.push()
def clear_ping(self, *args):
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
i.properties["ping_loss"] = 0
i.properties["ping_count"] = 1
i.push()
def ping_controllers(self, *args):
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control(
i.properties["vhost"])
ping_result = station_control.ping()
#print "ping_result",ping_result
if ping_result[0] == True:
i.properties["ping_count"] = float(
i.properties["ping_count"]) + 1
else:
i.properties["ping_count"] = float(
i.properties["ping_count"]) + 1
i.properties["ping_loss"] = float(
i.properties["ping_loss"]) + 1
i.push()
def clear_controller_resets(self, *args):
resets = {}
resets["temperature"] = False
resets["irrigation_resets"] = True
resets["system_resets"] = True
resets["clean_filter"] = False
resets["check_off"] = False
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
rpc_queue = i.properties["rpc_queue"]
redis_key = i.properties["redis_key"]
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control(
i.properties["vhost"])
redis_data = station_control.redis_hget_all([{
"hash": redis_key
}])[1][0]["data"]
for j in redis_data.keys():
if resets.has_key(j):
if resets[j] == True:
station_control.redis_hset([{
"hash": redis_key,
"key": j,
"data": 0
}])
def update_controller_properties(self, *args):
limits = {}
limits["temperature"] = {"yellow": 120, "red": 140}
limits["irrigation_resets"] = {"yellow": 1, "red": 2}
limits["system_resets"] = {"yellow": 1, "red": 2}
limits["ping"] = {"yellow": 1, "red": 10}
limits["clean_filter"] = self.process_clean_filter
limits["check_off"] = {"yellow": 2.0, "red": 3.0}
resets = {}
resets["temperature"] = False
resets["irrigation_resets"] = True
resets["system_resets"] = True
resets["clean_filter"] = False
resets["check_off"] = False
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
card_dict = json.loads(i.properties["card_dict"])
try:
ping_loss = float(i.properties["ping_loss"])
except:
ping_loss = 0
try:
ping_counts = float(i.properties["ping_counts"])
except:
ping_counts = 1
if ping_counts < 1:
ping_counts = 1
i.properties["ping_loss"] = 0
i.properties["ping_count"] = 1
i.push()
rate = ping_loss / ping_counts
card_name = card_dict["ping"]
card = self.qc.match_relation_property_specific(
"CONTROLLER", "name", i.properties["name"], "DIAGNOSTIC_CARD",
"name", card_name)[0]
self.update_properties(card, limits["ping"],
"RabbitMQ Packet Loss Rate", rate)
rpc_queue = i.properties["rpc_queue"]
redis_key = i.properties["redis_key"]
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control(
i.properties["vhost"])
redis_data = station_control.redis_hget_all([{
"hash": redis_key
}])[1][0]["data"]
for j in redis_data.keys():
if card_dict.has_key(j) == True:
card_name = card_dict[j]
card = self.qc.match_relation_property_specific(
"CONTROLLER", "name", i.properties["name"],
"DIAGNOSTIC_CARD", "name", card_name)[0]
self.update_properties(card, limits[j], j, redis_data[j])
if resets[j] == True:
station_control.redis_hset([{
"hash": redis_key,
"key": j,
"data": 0
}])
class Analyize_Remote_Connectivity:
def __init__( self, rabbit_interface, query_control ):
self.rc = rabbit_interface
self.qc = query_control
def convert_to_int( self, data ):
return_value = {}
for i in data.keys():
return_value[i] = int(data[i])
return return_value
def transform_data( self, data ):
data = self.convert_to_int( data )
error_rate = float(data["failures"])/float(data["counts"])
message_loss = data["total_failures"]
return error_rate, message_loss
def analyize_data( self,*args ):
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control( i.properties["vhost"] )
udp_server_list = self.qc.match_relation_property("CONTROLLER","name",i.properties["name"],"UDP_IO_SERVER")
for j in udp_server_list:
redis_key = j.properties["redis_key"]
data = station_control.redis_hget_all( [{"hash":redis_key } ])[1][0]["data"]
remote_list = self.qc.match_relation_property( "UDP_IO_SERVER","namespace", j.properties["namespace"],"REMOTE" )
for k in remote_list:
modbus_address = str(k.properties["modbus_address"])
remote_data = data[ modbus_address ]
error_rate, message_loss = self.transform_data( json.loads(remote_data) )
card_dict = json.loads( k.properties["card_dict"] )
card = self.qc.match_relation_property_specific( "CONTROLLER","namespace",i.properties["namespace"],"DIAGNOSTIC_CARD","name",card_dict["connectivity"] )[0]
text = "Controller: "+i.properties["name"]+ " Remote: "+k.properties["name"]+"\n"
if (error_rate < .03 ) and ( message_loss == 0 ):
text = text+"Error Rate: "+str(error_rate)+" Message Loss: "+str(message_loss)
color = "green"
elif message_loss == 0:
text = text+"**Error Rate: "+str(error_rate)+" Message Loss: "+str(message_loss)+ " ERROR **"
color = "yellow"
else:
text = text+"**Error Rate: "+str(error_rate)+" Message Loss: "+str(message_loss)+ " ERROR **"
color = "red"
try:
temp = json.loads( card.properties["new_commit"] )
if type(temp) is not list:
temp =[]
except:
temp = []
temp.append( text)
card.properties["new_commit"] = json.dumps(temp)
card.properties["label"] = color
card.push()
class Analyize_Controller_Parameters():
def __init__( self, rabbitmq_remote_connections, query_configuration ):
self.rc = rabbitmq_remote_connections
self.qc = query_configuration
def process_clean_filter( self, title,value ):
value = float(value)
ref = time.time()
hour_sec = 3600
if value > ref - 30*hour_sec:
color = "green"
text = "Title: "+str(title)+" Value: "+str((ref-value)/hour_sec)+" hours earlier "
elif value > ref-48*hour_sec:
color = "yellow"
text = "**Error Title: "+str(title)+" Value: "+str((ref-value)/hour_sec)+" hours earlier **"
else:
color = "red"
text = "**Error Title: "+str(title)+" Value: "+str((ref-value)/hour_sec)+" hours earlier **"
return color,text
def update_properties( self, card, limits, title,value ):
if hasattr(limits, '__call__') == True:
color , text = limits( title, value )
else:
value = float( value)
if value < limits["yellow"]:
color = "green"
text = "Title: "+str(title)+" Value: "+str(value)
elif value < limits["red"]:
color = "yellow"
text = "**Error Title: "+str(title)+" Value: "+str(value)+" **"
else:
color = "red"
text = "**Error Title: "+str(title)+" Value: "+str(value)+" **"
try:
temp = json.loads( card.properties["new_commit"] )
if type(temp) is not list:
temp = []
except:
temp = []
#print "title",title,value,color,text
temp.append( text)
card.properties["new_commit"] = json.dumps(temp)
card.properties["label"] = color
card.push()
def clear_ping( self, *args ):
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
i.properties["ping_loss"] = 0
i.properties["ping_count"] = 1
i.push()
def ping_controllers( self ,*args):
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control( i.properties["vhost"] )
ping_result = station_control.ping()
#print "ping_result",ping_result
if ping_result[0] == True:
i.properties["ping_count"] = float(i.properties["ping_count"] )+1
else:
i.properties["ping_count"] = float(i.properties["ping_count"] )+1
i.properties["ping_loss"] = float(i.properties["ping_loss"] )+1
i.push()
def clear_controller_resets( self,*args):
resets = {}
resets["temperature"] = False
resets["irrigation_resets"] = True
resets["system_resets"] = True
resets["clean_filter"] = False
resets["check_off"] = False
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
rpc_queue = i.properties["rpc_queue"]
redis_key = i.properties["redis_key"]
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control( i.properties["vhost"] )
redis_data = station_control.redis_hget_all( [ {"hash":redis_key} ] )[1][0]["data"]
for j in redis_data.keys():
if resets.has_key( j ):
if resets[j] == True:
station_control.redis_hset( [{"hash":redis_key ,"key":j,"data":0 }] )
def update_controller_properties( self,*args ):
limits = {}
limits["temperature"] = { "yellow":120, "red":140 }
limits["irrigation_resets"] = {"yellow":1,"red":2 }
limits["system_resets"] = {"yellow":1,"red":2 }
limits["ping"] = {"yellow":1,"red":10 }
limits["clean_filter"] = self.process_clean_filter
limits["check_off"] = {"yellow":2.0,"red":3.0 }
resets = {}
resets["temperature"] = False
resets["irrigation_resets"] = True
resets["system_resets"] = True
resets["clean_filter"] = False
resets["check_off"] = False
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
card_dict = json.loads( i.properties["card_dict"] )
try:
ping_loss = float(i.properties["ping_loss"])
except:
ping_loss = 0
try:
ping_counts = float(i.properties["ping_counts"])
except:
ping_counts = 1
if ping_counts < 1:
ping_counts = 1
i.properties["ping_loss"] = 0
i.properties["ping_count"] = 1
i.push()
rate = ping_loss/ping_counts
card_name = card_dict["ping"]
card = self.qc.match_relation_property_specific( "CONTROLLER","name",i.properties["name"],"DIAGNOSTIC_CARD","name",card_name )[0]
self.update_properties( card, limits["ping"],"RabbitMQ Packet Loss Rate", rate )
rpc_queue = i.properties["rpc_queue"]
redis_key = i.properties["redis_key"]
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control( i.properties["vhost"] )
redis_data = station_control.redis_hget_all( [ {"hash":redis_key} ] )[1][0]["data"]
for j in redis_data.keys():
if card_dict.has_key(j) == True:
card_name = card_dict[j]
card = self.qc.match_relation_property_specific( "CONTROLLER","name",i.properties["name"],"DIAGNOSTIC_CARD","name",card_name )[0]
self.update_properties( card, limits[j],j,redis_data[j] )
if resets[j] == True:
station_control.redis_hset( [{"hash":redis_key ,"key":j,"data":0 }] )
class Monitor_Event_Queues():
def __init__( self, rabbitmq_remote_connections, query_configuration, chain_flow ):
self.rc = rabbitmq_remote_connections
self.qc = query_configuration
self.cf = chain_flow
def process_card( self, card_dict, event_data ):
card_node = card_dict["card_node"]
label = card_dict["card_action"]["label"]
#print "event_data",event_data["event"]
#print "card action",card_dict["card_action"].keys()
if label == "fromevent":
label = event_data["status"].lower()
#print "---------->",label,event_data
diag_text = "New Event: "+event_data["event"] +" Data: "+json.dumps(event_data)
print "diag_text ",diag_text, label
try:
temp = json.loads( card_node.properties["new_commit"] )
if type(temp) is not list:
print "is not a list"
temp = []
except:
print "exception"
temp = []
temp.append(diag_text)
print "temp",temp
print "len",len(temp)
card_node.properties["new_commit"] = json.dumps(temp)
card_node.properties["label"] = label
card_node.push()
def get_cards( self, controller, cards_actions ):
return_value = {}
for i in cards_actions.keys():
card_name = cards_actions[i]["card"]
card = self.qc.match_relation_property_specific( "CONTROLLER","namespace", controller.properties["namespace"] ,"DIAGNOSTIC_CARD","name",card_name )
if len(card) > 0 :
#print "card_name",card_name
return_value[i] = { "card_node":card[0], "card_action": cards_actions[i] }
return return_value
def monitor_event_queue_cf( self, *args ):
self.monitor_event_queue( 50 )
def monitor_event_queue( self,search_depth, *args ):
ref_time = time.time()
self.rc = Rabbitmq_Remote_Connections()
controller_list = self.qc.match_labels("CONTROLLER")
for i in controller_list:
vhost = i.properties["vhost"]
self.rc = Rabbitmq_Remote_Connections()
station_control = self.rc.get_station_control( i.properties["vhost"] )
queue_list = self.qc.match_relation_property( "CONTROLLER","namespace", i.properties["namespace"],"EVENT_QUEUE" )
for j in queue_list:
cards_actions = json.loads(j.properties["events"] )
cards = self.get_cards( i, cards_actions )
redis_queue = j.properties["name"]
if j.properties["timestamp"] +48*3600 < time.time():
timestamp = time.time() - 48*3600
else:
timestamp = j.properties["timestamp"]
timestamp_max = timestamp
queue_length = int( station_control.redis_llen([redis_queue])[1][0]["data"] )
search_interval = range(0,search_depth)
search_interval.reverse()
for k in search_interval:
data = station_control.redis_lindex( [{"key":redis_queue, "index":k } ])
if data[0] == True:
data_base64 = data[1][0]["data"]
data_json = base64.b64decode( data_base64)
data = json.loads(data_json)
if data["time"] > timestamp_max:
timestamp_max = data["time"]
if timestamp < data["time"] :
#print k,time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(data["time"])), data["status"], data["event"]
event = data["event"]
if cards.has_key(event):
self.process_card( cards[event], data )
else:
pass
#print k,"no card match"
print "time_stamps",j.properties["timestamp"],timestamp_max
j.properties["timestamp"] = timestamp_max
j.push()
