def callback(self, conn, dom, event, detail, opaque):
myInstance = dom.name()
if myInstance.startswith('instance'):
global nodeName
nodeName = libvirt_utils.get_nova_name(myInstance)
global uuid
uuid = libvirt_utils.get_uuid_by_name(myInstance)
global tenant_id
tenant_id = libvirt_utils.get_tenant_id(myInstance)
if "instance" in dom.name():
if (time.time() - self.timeStamp_event4 > 30) and event == 4:
minor_id = 2032696
self.watchmenAlarm(True, region, tenant_id, uuid, nodeName,
minor_id,
enums.FM_ACTIVE_SEVERITY.CRITICAL,
"Virtual Machine OS Fault")
global timer
timer = threading.Timer(10.0, self.watchmenCleared,
[tenant_id, uuid, nodeName])
timer.start()
self.timeStamp_event4 = time.time()
elif (time.time() - self.timeStamp_event5 > 30) and event == 5:
minor_id = 2032697
self.watchmenAlarm(False, region, tenant_id, uuid,
nodeName, minor_id,
enums.FM_ACTIVE_SEVERITY.WARNING,
"Virtual Machine Running Fault")
self.timeStamp_event5 = time.time()
else:
self.timeStamp = time.time()
time.sleep(1)
def callback(self, conn, dom, event, detail, opaque):
print "EVENT: Domain %s(%s) %s %s" % (dom.name(), dom.ID(), event,
detail)
myInstance = dom.name()
nodeName = libvirt_utils.get_nova_name(myInstance)
uuid = dom.StringUUID()
tenant_id = libvirt_utils.get_tenant_id(myInstance)
logging.config.fileConfig('/var/cmcc-la/conf/libvirtevent_log.conf')
logger = logging.getLogger("libvirt_log")
timeStamp = 0
deltatime = time.time() - timeStamp
tstime = time.ctime()
if deltatime > 30 and "instance" in dom.name():
print 'instance in dom.name'
if event == 4: # reboot
print 'event=4'
minor_id = 2032696
timestatus, times = commands.getstatusoutput(
"cat /var/cmcc-la/data/libvirt-event-time.conf | grep $uuid | grep $minor_id | awk '{print $3}"
)
if times > 0: #~~~~~~~~~~~~~~~~~~thread
wa_timestampstatus, wa_timestamp = commands.getstatusoutput(
"date -d \"{} second ago\" +%Y-%m-%d\" \"%H:%M:%S.%6N -u"
.format(times))
logger.info("Adjust %s second ahead for %s,%s==>%s" %
(times, uuid, tstime, wa_timestamp))
self.watchmenAlarm(self.region, tenant_id, uuid, nodeName,
minor_id, enums.FM_ACTIVE_SEVERITY.CRITICAL,
"Virtual Machine OS Fault")
logger.error( #format watchmen-client create-event -sf -src Regio................... less the read stress
"watchmen-client create-event -sf -src Region=\"%s\",CeeFunction=\"1\",Tenant=\"%s\",VM=%s -ma 193 -mi 2032696 -s CRITICAL -e communicationsAlarm -p m3100Indeterminate -sp \"Virtual Machine OS Fault\" -t \"VM %s(%s)\""
% (self.region, tenant_id, uuid, nodeName, uuid))
logger.error(
'%s VM %s(%s): Detected Virtual Machine OS Fault,possible reason: kernel panic'
% (str(time.ctime()), nodeName, uuid))
time.sleep(10) #suggest timer
self.watchmenAlarm(self.region, tenant_id, uuid, nodeName,
minor_id, enums.FM_ACTIVE_SEVERITY.CLEARED,
"Virtual Machine OS Fault")
logger.info(
"watchmen-client create-event -sf -src Region=\"%s\",CeeFunction=\"1\",Tenant=\"%s\",VM=%s -ma 193 -mi 2032696 -s CLEARED -e communicationsAlarm -p m3100Indeterminate -sp \"Virtual Machine OS Fault\" -t \"VM %s(%s)\""
% (self.region, tenant_id, uuid, nodeName, uuid))
logger.info(
'%s VM %s(%s): Detected Virtual Machine OS Fault,possible reason: kernel panic.Cleared watchmen alarm.'
% (str(time.ctime()), nodeName, uuid))
elif event == 5: # down or destory
minor_id = 2032697
self.watchmenAlarm(self.region, tenant_id, uuid, nodeName,
minor_id, enums.FM_ACTIVE_SEVERITY.WARNING,
"Virtual Machine Running Fault")
logger.warning(
"watchmen-client create-event -sl -src Region=%s,CeeFunction=\"1\",Tenant=\"%s\",VM=$uuid -ma 193 -mi 2032697 -s WARNING -e communicationsAlarm -p m3100Indeterminate -sp \"Virtual Machine Running Fault\" -t \"VM %s(%s):%s\""
% (self.region, tenant_id, uuid, nodeName, uuid))
logger.warning(
'%s VM %s(%s): Detected Virtual Machine Running Fault,possible reason: qemu process had been killed or vm been powered off or shutdown abnormal.'
% (str(time.ctime()), nodeName, uuid))
else:
timeStamp = time.time()
time.sleep(1)
def callback(self, conn, dom, event, detail, opaque):
myInstance = dom.name()
if myInstance.startswith('instance'):
global nodeName
nodeName = libvirt_utils.get_nova_name(myInstance)
global uuid
uuid = libvirt_utils.get_uuid_by_name(myInstance)
global tenant_id
tenant_id = libvirt_utils.get_tenant_id(myInstance)
if "instance" in dom.name():
if (time.time() - self.timeStamp_event4 > 30) and event == 4:
minor_id = 2032696
self.watchmenAlarm(True, region, tenant_id, uuid, nodeName,
minor_id,
enums.FM_ACTIVE_SEVERITY.CRITICAL,
"Virtual Machine OS Fault")
logger.error(
"watchmen-client create-event -sf -src Region=\"%s\",CeeFunction=\"1\",Tenant=\"%s\","
"VM=%s -ma 193 -mi 2032696 -s CRITICAL -e communicationsAlarm -p m3100Indeterminate -sp \""
"Virtual Machine OS Fault\" -t \"VM %s(%s)\"" %
(region, tenant_id, uuid, nodeName, uuid))
logger.error(
'%s VM %s(%s): Detected Virtual Machine OS Fault,possible reason: kernel panic'
% (str(time.ctime()), nodeName, uuid))
global timer
timer = threading.Timer(10.0, self.watchmenCleared,
[tenant_id, uuid, nodeName])
timer.start()
self.timeStamp_event4 = time.time()
elif (time.time() - self.timeStamp_event5 > 30) and event == 5:
minor_id = 2032697
self.watchmenAlarm(False, region, tenant_id, uuid,
nodeName, minor_id,
enums.FM_ACTIVE_SEVERITY.WARNING,
"Virtual Machine Running Fault")
logger.warning(
"watchmen-client create-event -sl -src Region=%s,CeeFunction=\"1\",Tenant=\"%s\","
"VM=$uuid -ma 193 -mi 2032697 -s WARNING -e communicationsAlarm -p m3100Indeterminate -sp \""
"Virtual Machine Running Fault\" -t \"VM %s(%s):%s\"" %
(region, tenant_id, uuid, nodeName, uuid))
logger.warning(
'%s VM %s(%s): Detected Virtual Machine Running Fault,possible reason: '
'qemu process had been killed or vm been powered off or shutdown abnormal.'
% (str(time.ctime()), nodeName, uuid))
self.timeStamp_event5 = time.time()
else:
self.timeStamp = time.time()
time.sleep(1)
def callback(self, conn, dom, event, detail, opaque):
print "EVENT: Domain %s(%s) %s %s" % (dom.name(), dom.ID(), event,
detail)
myInstance = dom.name()
nodeName = libvirt_utils.get_nova_name(myInstance)
uuid = dom.StringUUID()
tenant_id = libvirt_utils.get_tenant_id(myInstance)
logging.config.fileConfig('/var/cmcc-la/conf/libvirtevent_log.conf')
logger = logging.getLogger("libvirt_log")
timeStamp = 0
deltatime = time.time() - timeStamp
tstime = time.ctime()
if deltatime > 30 and "instance" in dom.name():
if event == 4: # virsh reboot
minor_id = 2032696
timestatus, times = commands.getstatusoutput(
"cat /var/cmcc-la/data/libvirt-event-time.conf | grep $uuid | grep $minor_id | awk '{print $3}"
)
wa_timestamp = None
if times > 0:
wa_timestampstatus, wa_timestamp = commands.getstatusoutput(
"date -d \"{} second ago\" +%Y-%m-%d\" \"%H:%M:%S.%6N -u"
.format(times))
logger.info("Adjust %s second ahead for %s,%s==>%s" %
(times, uuid, tstime, wa_timestamp))
self.watchmenAlarm(self.region, tenant_id, uuid, nodeName,
minor_id, enums.FM_ACTIVE_SEVERITY.CRITICAL,
"Virtual Machine OS Fault")
# evnet_sender = EventSender()
# source_one = "Region=%s,CeeFunction=1,Tenant=%s,VM=%s" % (
# self.region, tenant_id, uuid)
# event = FmEvent(True, source_one, 193, 2032696,
# enums.FM_ACTIVE_SEVERITY.CRITICAL, enums.FM_EVENT_TYPE.communicationsAlarm,
# enums.FM_PROBABLE_CAUSE.m3100Indeterminate,
# "Virtual Machine OS Fault",
# None,
# "VM %s(%s):$myReason" % (nodeName, uuid))
# evnet_sender.create_new_fm_event(event)
logger.critical(
"watchmen-client create-event -sf -src Region=\"%s\",CeeFunction=\"1\",Tenant=\"%s\",VM=%s -ma 193 -mi 2032696 -s CRITICAL -e communicationsAlarm -p m3100Indeterminate -sp \"Virtual Machine OS Fault\" -t \"VM %s(%s)\""
% (self.region, tenant_id, uuid, nodeName, uuid))
logger.critical(
'%s VM %s(%s): Detected Virtual Machine OS Fault,possible reason: kernel panic'
% (str(time.ctime()), nodeName, uuid))
if wa_timestamp:
commands.getstatusoutput(
"python /var/cmcc-la/scripts/watchmen-fmevent-operate.py watchmen %s %s %s"
% (minor_id, uuid, wa_timestamp))
time.sleep(10)
self.watchmenAlarm(self.region, tenant_id, uuid, nodeName,
minor_id, enums.FM_ACTIVE_SEVERITY.CLEARED,
"Virtual Machine OS Fault")
# evnet_sender = EventSender()
# source_one = "Region=%s,CeeFunction=1,Tenant=%s,VM=%s" % (
# region, tenant_id, uuid)
# event = FmEvent(True, source_one, 193, 2032696,
# enums.FM_ACTIVE_SEVERITY.CLEARED, enums.FM_EVENT_TYPE.communicationsAlarm,
# enums.FM_PROBABLE_CAUSE.m3100Indeterminate,
# "Virtual Machine OS Fault",
# None,
# "VM %s(%s):$myReason" % (nodeName, uuid))
# evnet_sender.create_new_fm_event(event)
logger.info(
"watchmen-client create-event -sf -src Region=\"%s\",CeeFunction=\"1\",Tenant=\"%s\",VM=%s -ma 193 -mi 2032696 -s CLEARED -e communicationsAlarm -p m3100Indeterminate -sp \"Virtual Machine OS Fault\" -t \"VM %s(%s)\""
% (self.region, tenant_id, uuid, nodeName, uuid))
logger.info(
'%s VM %s(%s): Detected Virtual Machine OS Fault,possible reason: kernel panic.Cleared watchmen alarm.'
% (str(time.ctime()), nodeName, uuid))
elif event == 5: # shutoff virsh shutdown ,virsh destory
minor_id = 2032697
self.watchmenAlarm(self.region, tenant_id, uuid, nodeName,
minor_id, enums.FM_ACTIVE_SEVERITY.WARNING,
"Virtual Machine Running Fault")
# evnet_sender = EventSender()
# source_one = "Region=%s,CeeFunction=1,Tenant=%s,VM=%s" % (
# region, tenant_id, uuid) # k:vm_uuid
# event = FmEvent(True, source_one, 193, 2032697,
# enums.FM_ACTIVE_SEVERITY.WARNING, enums.FM_EVENT_TYPE.communicationsAlarm,
# enums.FM_PROBABLE_CAUSE.m3100Indeterminate,
# "Virtual Machine Running Fault",
# None,
# "VM %s(%s):$myReason" % (nodeName, uuid))
# evnet_sender.create_new_fm_event(event)
logger.warning(
"watchmen-client create-event -sl -src Region=%s,CeeFunction=\"1\",Tenant=\"%s\",VM=$uuid -ma 193 -mi 2032697 -s WARNING -e communicationsAlarm -p m3100Indeterminate -sp \"Virtual Machine Running Fault\" -t \"VM %s(%s):%s\""
% (self.region, tenant_id, uuid, nodeName, uuid))
logger.critical(
'%s VM %s(%s): Detected Virtual Machine Running Fault,possible reason: qemu process had been killed or vm been powered off or shutdown abnormal.'
% (str(time.ctime()), nodeName, uuid))
else:
timeStamp = time.time()
time.sleep(1)