class Main():
def __init__(self):
self.messageHandler = MessageHandler()
self.communicator = Communicator(self.messageHandler)
self.controller = Controller(self.communicator)
self.messageHandler.setController(self.controller)
print "MAIN initialize"
## test code.
print "--------------"
sleep(2)
print "testing start"
peer = Peer("78.91.5.10")
message = Message("stillAlive", peer.IP, "", "", "", "")
self.communicator.broadcast(message)
#self.communicator.sendToOne(peer, message)
print "testing complete"
print "--------------"
def handle_instance(self, instance, msg_type, msg):
global jobs
global job_lock
if msg_type == "READY":
try:
job_lock.acquire()
if len(jobs) == 0:
logger.info("No jobs to assign! Terminating %s" % (INSTANCE_ID))
self.job = "TERMINATE"
else:
self.job = jobs[0] + "\n"
jobs = jobs[1:]
self.wfile.write(self.job)
messenger = MessageHandler(MASTER_IP, MASTER_PORT)
messenger.sendMessage("%s,%s:JOB_ASSIGN:%s,%s" % (INSTANCE_ID, PUBLIC_HOSTNAME, instance, self.job))
messenger.close()
logger.info(self.job)
finally:
job_lock.release()
if msg_type == "DATA":
logger.info(msg)
if msg_type == "FINISH":
logger.info("Job complete: " + self.job)
messenger = MessageHandler(MASTER_IP, MASTER_PORT)
messenger.sendMessage("%s,%s:JOB_COMPLETE:%s,%s" % (INSTANCE_ID, PUBLIC_HOSTNAME, instance, self.job))
messenger.close()
def callback(ch, method, properties, body):
print " [x] Received %r" % (body,)
print " [x] Done"
# creating message dict from string then we can process that
msg = eval(body)
if isinstance(msg, dict):
mh = MessageHandler()
mh.processMsg(msg)
else:
print "unsupported message: Can't be processed"
ch.basic_ack(delivery_tag = method.delivery_tag)
def callback(ch, method, properties, body):
print " [x] Received %r" % (body, )
print " [x] Done"
# creating message dict from string then we can process that
msg = eval(body)
if isinstance(msg, dict):
mh = MessageHandler()
mh.processMsg(msg)
else:
print "unsupported message: Can't be processed"
ch.basic_ack(delivery_tag=method.delivery_tag)
def finalizeCodeOutput(self, command, output, workingDir):
"""
this method is called by the RAVEN code at the end of each run (if the method is present, since it is optional).
It can be used for those codes, that do not create CSV files to convert the whatever output formats into a csv
@ In, command, string, the command used to run the just ended job
@ In, output, string, the Output name root
@ In, workingDir, string, current working dir
@ Out, dataObjectsToReturn, dict, optional, this is a special case for RAVEN only. It returns the constructed dataobjects
(internally we check if the return variable is a dict and if it is returned by RAVEN (if not, we error out))
"""
##### TODO This is an exception to the way CodeInterfaces usually run.
# The return dict for this CodeInterface is a dictionary of data objects (either one or two of them, up to one each point set and history set).
# Normally, the end result of this method is producing a CSV file with the data to load.
# When data objects are returned, this triggers an "if" path in Models/Code.py in evaluateSample(). There's an "else"
# that can be found by searching for the comment "we have the DataObjects -> raven-runs-raven case only so far".
# See more details there.
#####
dataObjectsToReturn = {}
numRlz = None
messageHandler = MessageHandler()
messageHandler.initialize({'verbosity': 'quiet'})
for filename in self.linkedDataObjectOutStreamsNames:
# load the output CSV into a data object, so we can return that
## load the XML initialization information and type
dataObjectInfo = self.outStreamsNamesAndType[filename]
# create an instance of the correct data object type
data = DataObjects.returnInstance(dataObjectInfo[1], None)
# initialize the data object by reading the XML
data.readXML(dataObjectInfo[2],
messageHandler,
variableGroups=self.variableGroups)
# set the name, then load the data
data.name = filename
data.load(os.path.join(workingDir, self.innerWorkingDir, filename),
style='csv')
# check consistency of data object number of realizations
if numRlz is None:
# set the standard if you're the first data object
numRlz = len(data)
else:
# otherwise, check that the number of realizations is appropriate
if len(data) != numRlz:
raise IOError(
'The number of realizations in output CSVs from the inner RAVEN run are not consistent! In "{}" received "{}" realization(s), but other data objects had "{}" realization(s)!'
.format(data.name, len(data), numRlz))
# store the object to return
dataObjectsToReturn[dataObjectInfo[0]] = data
return dataObjectsToReturn
def turn_on(self):
print("System turning on")
self.message_handler = MessageHandler(
self.MESSAGE_HANDLER_MAX_IDLE_TIME, self.bot_controller_factory,
self.debug_mode)
self.sending_manager = SendingManager(self.SENDING_MANAGER_SLEEP_TIME,
self.bot_controller_factory,
self.debug_mode)
self.learnit = LearnIt(self.message_handler, self.sending_manager,
self.bot_controller_factory)
self.learnit.start()
print("System turned on")
def __init__(self, listenport, identifier=""):
self.serverPort = listenport
self.acceptAddress = 'localhost'
self.id = identifier
self.running = False
self.connected = False
self.handledData = list()
self.receiveBuffer = list()
self.sendBuffer = list()
self.receivingThread = Thread(target=self.receivingLoop)
self.sendingThread = Thread(target=self.sendingLoop)
self.messageHandler = MessageHandler()
self.listenSocket = socket(AF_INET, SOCK_STREAM)
self.listenSocket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.first = True
self.clientSocket = None
self.clientAddress = None
self.sending_timer = 0.1 # minska för att öka antalet meddelanden per sekund
self.last_send = 0
def turn_on(self):
print("System turning on")
self.bot_controller_factory = BotControllerFactory(self.TOKEN)
self.message_handler = MessageHandler(self.bot_controller_factory,
self.debug_mode)
self.sending_manager = SendingManager(self.SENDING_MANAGER_SLEEP_TIME,
self.bot_controller_factory,
self.debug_mode)
self.learnit = LearnItThread(self.LEARNIT_SLEEP, self.message_handler,
self.sending_manager,
self.bot_controller_factory)
self.learnit.start()
print("System turned on")
def __init__(self):
"""
Constructor.
@ In, None
@ Out, None
"""
Base.__init__(self)
self._components = [] # units involved in this study
self._sources = [
] # sources involved in this study (arma, functions, static histories, etc)
self._case = None # information about the case we're running
self._input_dir = None # location of the input XML file
self._input_name = None # name of the input XML file
messageHandler = MessageHandler()
messageHandler.initialize({
'verbosity': 'debug',
'callerLength': 25,
'tagLength': 15,
'suppressErrs': False
})
self.messageHandler = messageHandler
def __init__(self):
self.messageHandler = MessageHandler()
self.communicator = Communicator(self.messageHandler)
self.controller = Controller(self.communicator)
self.messageHandler.setController(self.controller)
print "MAIN initialize"
## test code.
print "--------------"
sleep(2)
print "testing start"
peer = Peer("78.91.5.10")
message = Message("stillAlive", peer.IP, "", "", "", "")
self.communicator.broadcast(message)
#self.communicator.sendToOne(peer, message)
print "testing complete"
print "--------------"
def __init__(self):
"""
"""
self.getOptions()
#self.options.verbose = True
self.zenoss = ZenossHandler(self.options.zenhost, self.options.zenuser,
self.options.zenpass, self.options.verbose)
self.pagerduty = PagerDutyHandler(self.options.pdhost,
self.options.pdtoken,
self.options.verbose)
# dictionary of corresponding event/incident states
self.statusData = {
"ack": {
"zenoss": "Acknowledged",
"pagerduty": "acknowledged",
"zenaction": "acknowledge",
"num": 1
},
"close": {
"zenoss": "Closed",
"pagerduty": "resolved",
"zenaction": "close",
"num": 2
},
"open": {
"zenoss": "New",
"pagerduty": "triggered",
"zenaction": "unacknowledge",
"num": 0
},
}
if self.options.zenver == True:
self.statusData["open"]["zenaction"] = "reopen"
self.messenger = MessageHandler()
self.statusDict = {}
def setup_module(module):
global messageHandler
global rawInput1
global rawInput2
global rawInput3
messageHandler = MessageHandler(Queue.Queue(), Queue.Queue(), botID, False)
rawInput1 = [{
'text': "Hello <@" + botID + "> my name is tester",
'user': '******',
'team': 'TESTTESM123',
'channel': 'TESTCHANNEL123',
'bot_id': 'TESTBOT123',
'ts': '1524798523.000234',
'type': 'message',
'event_ts': '1524798523.000234'
}]
rawInput2 = [{
'event_ts': '1527524745.000005',
'msg': '1527524744.000299',
'imageUri': None,
'content': 'Hello',
'type': 'desktop_notification',
'subtitle': 'J Dolla',
'launchUri': 'slack://channel?id=TESTCHANNEL123&message=123&team=TEAM',
'avatarImage': 'https://avatars.slack-edge.com/file.png',
'ssbFilename': 'knock_brush.mp3',
'title': 'SLACK_NAME',
'channel': 'TESTCHANNEL123',
'is_shared': False
}]
rawInput3 = [{
'avatarImage': 'someImage.png',
'launchUri': 'slack://channel?id=someId',
'imageUri': None,
'subtitle': 'J Dolla',
'event_ts': '1527526728.000092',
'channel': 'TESTCHANNEL123',
'title': 'SLACK_NAME',
'ssbFilename': 'file.mp3',
'is_shared': False,
'type': 'desktop_notification',
'content': 'Hello',
'msg': '1527526728.000025'
}]
def __init__(self):
"""
"""
self.getOptions()
#self.options.verbose = True
self.zenoss = ZenossHandler(self.options.zenhost, self.options.zenuser, self.options.zenpass, self.options.verbose)
self.pagerduty = PagerDutyHandler(self.options.pdhost, self.options.pdtoken, self.options.verbose)
# dictionary of corresponding event/incident states
self.statusData = {
"ack":{"zenoss":"Acknowledged","pagerduty":"acknowledged","zenaction":"acknowledge","num":1},
"close":{"zenoss":"Closed","pagerduty":"resolved","zenaction":"close","num":2},
"open":{"zenoss":"New","pagerduty":"triggered","zenaction":"unacknowledge","num":0},
}
if self.options.zenver == True:
self.statusData["open"]["zenaction"] = "reopen"
self.messenger = MessageHandler()
self.statusDict = {}
def __init__(self, config_file):
# setup configuration
self.config = ConfigParser.RawConfigParser()
self.config.read(config_file)
self.user_nickname = self.config.get(CONFIG_AUTH, 'user_nickname')
self.user_jid = self.config.get(CONFIG_AUTH, 'user_jid')
self.user_pwd = self.config.get(CONFIG_AUTH, 'user_pwd')
self.user_api_token = self.config.get(CONFIG_AUTH, 'user_api_token')
# setup xmpp client
ClientXMPP.__init__(self, self.user_jid, self.user_pwd)
self.add_event_handler("session_start", self.session_start)
self.add_event_handler("message", self.message)
# register plugins for additional functionality
self.register_plugin('xep_0030') # Service Discovery
self.register_plugin('xep_0004') # Data Forms
self.register_plugin('xep_0045') # MUC
self.register_plugin('xep_0060') # PubSub
self.register_plugin('xep_0199') # Ping
# Setup message handler
self.msg_handler = MessageHandler(self)
# Setup HypChat api client
self.hc = HypChat(self.user_api_token)
# get jid to room map
self.jid_to_room = dict()
# Join rooms on startup
startup_rooms = self.config.get(CONFIG_GENERAL, 'startup_rooms_to_join').split(',')
for room in startup_rooms:
self.join_room_by_name(room)
print('Bot initialized')
def startConnections():
global amqpConnection
amqpConnection = newAmqpConnection(config)
global mysqlConnection
mysqlConnection = DatabaseConnection(
MySQLdb.connect(host=config.dbHost,
user=config.dbUser,
passwd=config.dbPass,
db="upsilon",
connect_timeout=5,
autocommit=True))
messageHandler = MessageHandler(amqpConnection, mysqlConnection, config)
amqpConnection.addMessageTypeHandler("GET_LIST", messageHandler.onGetList)
amqpConnection.addMessageTypeHandler("GET_ITEM", messageHandler.onGetItem)
amqpConnection.addMessageTypeHandler("HEARTBEAT",
messageHandler.onHeartbeat)
amqpConnection.addMessageTypeHandler("SERVICE_CHECK_RESULT",
messageHandler.onServiceCheckResult)
logger.info("AMQP and MySQL are connected, consuming")
amqpConnection.startConsuming()
def __init__(self, zenhost, zenuser, zenpass, pdhost, pdtoken, pduser, verbose=False):
''''''
self.zenhost = zenhost
self.zenuser = zenuser
self.zenpass = zenpass
self.pdhost = pdhost
self.pdtoken = pdtoken
self.pduser = pduser
self.logs = [] # array to hold log messages
self.verbose = verbose
self.buffersize = 100
self.statusData = {
"ack": {
"zenoss":"Acknowledged",
"pagerduty":"acknowledged",
"zenaction":"acknowledge",
"num":1
},
"close":{
"zenoss":"Closed",
"pagerduty":"resolved",
"zenaction":"close",
"num":2
},
"open":{
"zenoss":"New",
"pagerduty":"triggered",
"zenaction":"reopen",
"num":0},
}
self.zenver = self.isVersion4()
if self.zenver is True: #zenoss 4 changed "unacknowledge" action to "reopen"
self.statusData["open"]["zenaction"] = "reopen"
self.messenger = MessageHandler()
self.statusDict = {}
self.commonkeys = []
from Elevator import Elevator
from Communicator import Communicator
from MessageHandler import MessageHandler
from Controller import Controller
from Message import Message
from Peer import Peer
#Elevator()
messageHandler = MessageHandler()
communicator = Communicator(messageHandler)
controller = Controller(communicator)
print "main"
message = Message("newOrder")
print message
peer = Peer("129.241.187.145")
#communicator.broadcast(message)
#communicator.sendToOne(peer, message)
print "main end"
class TcpServer:
def __init__(self, listenport, identifier=""):
self.serverPort = listenport
self.acceptAddress = 'localhost'
self.id = identifier
self.running = False
self.connected = False
self.handledData = list()
self.receiveBuffer = list()
self.sendBuffer = list()
self.receivingThread = Thread(target=self.receivingLoop)
self.sendingThread = Thread(target=self.sendingLoop)
self.messageHandler = MessageHandler()
self.listenSocket = socket(AF_INET, SOCK_STREAM)
self.listenSocket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.first = True
self.clientSocket = None
self.clientAddress = None
self.sending_timer = 0.1 # minska för att öka antalet meddelanden per sekund
self.last_send = 0
def reset(self):
print("[" + self.id + "] Resetting")
if self.running:
self.stop()
self.running = False
self.connected = False
self.handledData.clear()
self.receiveBuffer.clear()
self.sendBuffer.clear()
self.receivingThread = Thread(target=self.receivingLoop)
self.sendingThread = Thread(target=self.sendingLoop)
self.listenSocket = socket(AF_INET, SOCK_STREAM)
self.listenSocket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
# open a listening socket on set port to listen for client connections
def connect(self):
print("[" + self.id + "] Waiting for connection")
try:
self.listenSocket.bind((self.acceptAddress, self.serverPort))
self.listenSocket.listen(1)
except OSError as e:
print(e)
self.disconnect()
return False
try:
(self.clientSocket, self.clientAddress) = self.listenSocket.accept()
except Exception as e:
print("[" + self.id + "]" + str(e) + " Stopped listening")
self.disconnect()
return False
#self.listenSocket.shutdown(SHUT_RDWR)
#self.listenSocket.close()
print("[" + self.id + "] Connection established - Client: " + str(self.clientAddress))
self.connected = True
return True
def isRunning(self):
return self.running
def isConnected(self):
return self.connected
# TODO fix disconnect, client needs to know server is disconnecting
def disconnect(self):
try:
self.clientSocket.shutdown(SHUT_RDWR)
except Exception as e:
print("["+self.id+"][disconnect1]"+str(e))
try:
self.clientSocket.close()
except Exception as e:
print("["+self.id+"][disconnect2]"+str(e))
try:
self.listenSocket.shutdown(SHUT_RDWR)
except Exception as e:
print("["+self.id+"][disconnect3]"+str(e))
try:
self.listenSocket.close()
except Exception as e:
print("["+self.id+"][disconnect4]"+str(e))
self.connected = False
# set which IP addresses the server accepts
def setAcceptAddress(self, ip):
self.acceptAddress = ip
# set which port the server listens on
def setServerPort(self, port):
self.serverPort = port
# main server loop
def run(self):
while self.running:
if self.receiveBuffer:
data = self.receiveBuffer.pop(0)
handledData = self.messageHandler.handle(data) # Returns touple with (handledData, response to client)
if handledData:
self.handledData.append(handledData)
self.stop()
# Loop for sending thread
def sendingLoop(self):
while self.running:
if self.sendBuffer:
try:
now = time()
if now - self.last_send > self.sending_timer:
data = self.sendBuffer.pop(0)
#print("Data start send by server: " + str(data) + " " + str(datetime.datetime.now()))
self.clientSocket.sendall(data.encode('UTF-8'))
#print("Data done send by server: " + str(data) + " " + str(datetime.datetime.now()))
self.last_send = now
except Exception as e:
print(e)
self.running = False
self.sendBuffer = list()
# Loop for receiving thread
def receivingLoop(self):
while self.running:
try:
data = self.clientSocket.recv(1024).decode('UTF-8')
#print("Data done received by server: " + str(data) + " " + str(datetime.datetime.now()))
if not data:
self.running = False
else:
self.receiveBuffer.append(data)
except ConnectionAbortedError:
self.running = False
except Exception as e:
print(e)
self.running = False
# Method to be able to send data from outside class outside
def send(self, data):
self.sendBuffer.append(data)
# get data handled with defined messagehandler
def getHandledData(self):
if self.handledData:
return self.handledData.pop(0)
return False
# check if server has received data
def isDataAvailable(self):
if self.handledData:
return True
return False
def stopServer(self):
self.running = False
self.disconnect()
# stops the server
def stop(self):
self.disconnect()
if self.sendingThread.isAlive():
self.sendingThread.join()
if self.receivingThread.isAlive():
self.receivingThread.join()
print("[" + self.id + "] Disconnected and stopped")
# starts server
def start(self):
self.reset()
self.running = True
self.connect()
if self.isConnected():
self.sendingThread.start()
self.receivingThread.start()
self.run()
self.running = False
def setMessageHandler(self, messageHandler):
self.messageHandler = messageHandler
def clearHandledBuffer(self):
self.handledData.clear()
def clearSendingBuffer(self):
self.sendBuffer.clear()
def clearReceivingBuffer(self):
self.receiveBuffer.clear()
class Main():
"""
Either:
1) Create Pagerduty Incident based on new Zenoss event
2) Synchronize Zenoss and Pagerduty Events and Incidents
"""
def __init__(self):
"""
"""
self.getOptions()
#self.options.verbose = True
self.zenoss = ZenossHandler(self.options.zenhost, self.options.zenuser, self.options.zenpass, self.options.verbose)
self.pagerduty = PagerDutyHandler(self.options.pdhost, self.options.pdtoken, self.options.verbose)
# dictionary of corresponding event/incident states
self.statusData = {
"ack":{"zenoss":"Acknowledged","pagerduty":"acknowledged","zenaction":"acknowledge","num":1},
"close":{"zenoss":"Closed","pagerduty":"resolved","zenaction":"close","num":2},
"open":{"zenoss":"New","pagerduty":"triggered","zenaction":"unacknowledge","num":0},
}
if self.options.zenver == True:
self.statusData["open"]["zenaction"] = "reopen"
self.messenger = MessageHandler()
self.statusDict = {}
def lastDisabled(self,service):
"""
return PagerDuty newest disabled entry
"""
message = ""
details = self.pagerduty.getServiceLog(service["id"])["log_entries"]
last = 0
lastentry = None
for d in details:
if d["maintenance_window"]["type"] == "disable":
starttime = self.messenger.getLocalTime(d["maintenance_window"]["time"])
start = self.messenger.getTimestamp(starttime)
if start > last:
lastentry = d
last = start
return lastentry
def getMaintenanceWindows(self,service):
"""
return list of maintenance windows (ongoing) for a given service
"""
output = []
windows = self.pagerduty.getMaintenanceWindows()["maintenance_windows"]
for w in windows:
services = w["services"]
for s in services:
if s["id"] == service["id"]:
beg = self.messenger.getLocalTime(w["start_time"])
fin = self.messenger.getLocalTime(w["end_time"])
start = self.messenger.getTimestamp(beg)
end = self.messenger.getTimestamp(fin)
now = time.time()
if start <= now and now <=end:
output.append(w)
return output
def createIncidentDetails(self,evid):
"""
Retrieve event detail from Zenoss and format it
for PagerDuty incident creation
"""
self.incidentData = {
"service_key": self.options.servicekey,
"incident_key": self.options.evid
}
details = self.zenoss.getEventDetails(evid)["result"]["event"][0]["properties"]
self.statusDict[evid] = {}
self.statusDict[evid]["target"] = "open"
data = {}
status = 0
dev = None
comp = None
summ = None
for d in details:
k = d["key"]
v = d["value"]
data[k] = v
if k == "device": dev=v
if k == "component": comp = v
if k == "summary": summ = v
if k == "eventState": status = v
self.incidentData["details"] = data
self.incidentData["description" ] = "%s | %s | %s" % (dev, comp, summ)
if status == 0:
self.statusDict[evid]["current"] = "open"
elif status == 1:
self.statusDict[evid]["current"] = "ack"
elif status == 2:
self.statusDict[evid]["current"] = "close"
return status
def createPagerDutyIncident(self):
"""
1) check whether the destination service is in a maintenance window.
a) If so, ack the local alert in zenoss
b) and add a "in maintenance window" log message to the Zenoss event details.
2) if it is not in maintenance, proceed with the event submission. as usual
a) send event to pagerduty
b) update Zenoss console with submit status info
c) update Zenoss Event with incident details (incident URL, service URL, oncall)
3) check issues in Zenoss w/pagerduty incidents:
a) if acked in Zenoss, ack in PagerDuty
b) if closed in Zenoss, resolve in PagerDuty
"""
status = self.createIncidentDetails(self.options.evid)
if status == 0:
# first find the appropriate PD service definition
service = self.pagerduty.findService(self.options.servicekey)
if service:
# in maintenance, so ack the zenoss alert but note the window detail
if self.pagerduty.inMaintenance(service) == True:
self.statusDict[self.options.evid]["target"] = "ack"
mws = self.getMaintenanceWindows(service)
for mw in mws:
self.messenger.serviceInMaintenance(self.options.evid, "Acknowledged", service, mw, self.pagerduty.weburl)
# disabled, so leave event unacked in Zenoss, but that service is disabled
elif self.pagerduty.isDisabled(service) == True:
self.messenger.serviceIsDisabled(self.options.evid, "No Incident created", service, self.lastDisabled(service), self.pagerduty.weburl)
# assuming service is enabled, create PD incident, note output in zenoss event console.
else:
output = self.pagerduty.manageIncident(self.incidentData,"trigger")
try:
self.messenger.serviceIncidentCreated(self.options.evid, service, self.pagerduty.weburl, output["errors"])
except KeyError:
self.messenger.serviceIncidentCreated(self.options.evid, service, self.pagerduty.weburl)
else:
self.messenger.serviceNotFound(self.options.evid, self.options.servicekey)
else:
self.statusDict[self.options.evid]["target"] = self.statusDict[self.options.evid]["current"]
def correlateByZenoss(self):
"""
build dictionary of Zenoss events matched to PagerDuty incidents
1) get list of zenoss events
2) get list of pagerduty incidents based on date range of zenoss events
3) match them by evid - incident_key
"""
if self.options.verbose == True:
print "CORRELATING ZENOSS EVENTS AND PAGERDUTY INCIDENTS"
#incidents = self.pagerduty.getIncidentList()["incidents"]
events = self.zenoss.getEvents()["events"]
self.pairs = []
for e in events:
data = {"id": e["evid"], 'pagerduty': None, 'zenoss': e}
try:
data["pagerduty"]= self.pagerduty.getIncidentByKey(e["evid"])
except:
pass
self.pairs.append(data)
def correlateByPagerDuty(self):
"""
build dictionary of Zenoss events matched to PagerDuty incidents
1) get list of zenoss events
2) get list of pagerduty incidents based on date range of zenoss events
3) match them by evid - incident_key
"""
if self.options.verbose == True:
print "CORRELATING ZENOSS EVENTS AND PAGERDUTY INCIDENTS"
incidents = self.pagerduty.getIncidentList()["incidents"]
events = self.zenoss.getEvents()["events"]
self.pairs = []
for i in incidents:
data = {"id": i["incident_key"], 'pagerduty': i, 'zenoss': None}
for e in events:
if i["incident_key"] == e["evid"]:
data['zenoss'] = e
self.pairs.append(data)
def synchronize(self):
"""
update pagerduty based on Zenoss event console activities (acknowledge, resolve, etc)
4) if acked in one, ack in the other
5) if closed in one, close in the other
"""
if self.options.verbose == True:
print "SYNCHRONIZING ZENOSS AND PAGERDUTY EVENT/INCIDENT STATES"
for p in self.pairs:
id = p["id"]
zen = p["zenoss"]
pd = p["pagerduty"]
self.messenger.newId(id)
self.statusDict[id] = {}
# if pagerduty event is open, but not listed in zenoss, then close it (for Zenoss 3.x, to avoid querying event history)
if zen == None:
#if self.options.zenver == False:
self.statusDict[id]["target"] = "close"
else:
# set up console messages
self.messenger.incidentCreated(id, pd)
#self.messenger.incidentAssigned(id, pd)
details = self.pagerduty.getIncidentDetail(pd["id"])["log_entries"]
self.messenger.incidentLogs(id, details, self.pagerduty.weburl)
for k,v in self.statusData.items():
if v["zenoss"] == zen["eventState"]: self.statusDict[id]["current"] = k
# determine whether Zenoss or Pagerduty has authority
self.compare(zen, pd)
def compare(self,event,incident):
"""
determine target state based on event vs. incident last updated
"""
if self.options.verbose == True:
print "COMPARING EVENT AND INCIDENT FOR ID:%s" % event["evid"]
evupdated = None
evdetails = details = self.zenoss.getEventDetails(event["evid"])["result"]["event"][0]["properties"]
for e in evdetails:
k = e["key"]
v = e["value"]
if k == "stateChange": eupdated = v
iupdated = incident["last_status_change_on"]
zt = self.messenger.getZenossTime(eupdated)
pt = self.messenger.getUTCTime(iupdated)
if self.options.verbose == True:
print "COMPARING zenoss time %s and pagerduty time %s" % (eupdated,iupdated)
print "COMPARING zenoss time %s and pagerduty time %s" % (zt,pt)
zentime = self.messenger.getTimestamp(zt)
pdtime = self.messenger.getTimestamp(pt)
zenstate = event['eventState']
pdstate = incident['status']
if self.options.verbose == True:
print "EVENT ID %s IS %s IN ZENOSS, %s IN PAGERDUTY" % (event["evid"],zenstate,pdstate)
zentarget = None
pdtarget = None
for k,v in self.statusData.items():
if v["zenoss"] == zenstate:
zentarget = k
if v["pagerduty"] == pdstate:
pdtarget = k
if zentime >= pdtime:
if self.options.verbose == True:
print "ZENOSS %s NEWER THAN PAGERDUTY %s" % (zenstate,pdstate)
self.statusDict[event["evid"]]["target"] = zentarget
else:
if self.options.verbose == True:
print "PAGERDUTY %s NEWER THAN ZENOSS %s" % (pdstate,zenstate)
self.statusDict[event["evid"]]["target"] = pdtarget
def getIncidentLogs(self,id,evid):
details = self.pagerduty.getIncidentDetail(id)["log_entries"]
self.messenger.incidentLogs(evid, details, self.pagerduty.weburl)
def updatePagerDuty(self):
"""
update PagerDuty incident status
"""
if self.options.verbose == True:
print "UPDATING PAGERDUTY"
updates = {"incidents": [], "requester_id": self.options.pduser}
for p in self.pairs:
id = p["id"]
pd = p["pagerduty"]
target = self.statusDict[id]["target"]
if pd["status"] != self.statusData[target]["pagerduty"]:
if self.options.verbose == True:
print "INCIDENT %s IS %s; SHOULD BE %s" % (pd["id"],pd["status"],self.statusData[target]["pagerduty"])
pdData = {"id": pd["id"], "status": self.statusData[target]["pagerduty"]}
updates["incidents"].append(pdData)
if len(updates["incidents"]) > 0:
self.pagerduty.updateStatus(updates)
def updateZenoss(self):
"""
update Zenoss console
"""
if self.options.verbose == True:
print "UPDATING ZENOSS"
for p in self.pairs:
id = p["id"]
zen = p["zenoss"]
if p["zenoss"] != None:
self.updateZenossMessages(id)
for p in self.pairs:
id = p["id"]
zen = p["zenoss"]
if p["zenoss"] != None:
self.updateZenossStatus(id)
def updateZenossStatus(self,evid):
"""
update Zenoss event status if target is not current
"""
current = self.statusDict[evid]["current"]
target = self.statusDict[evid]["target"]
if current != target:
if self.options.verbose == True:
print "CHANGING STATUS %s IN ZENOSS TO %s" % (evid,target)
self.zenoss.manageEventStatus([evid], self.statusData[target]['zenaction'])
def updateZenossMessages(self,evid):
"""
update Zenoss event console with messages if they don't exist
"""
eventlog = self.zenoss.getEventDetails(evid)["result"]["event"][0]["log"]
logs = []
for e in eventlog:
logs.append(e[-1])
for msg in self.messenger.messages[evid]:
if msg not in logs:
self.zenoss.addEventMessage(evid,msg)
def getOptions(self):
"""
Command line runtime arguments
"""
usage = "usage: %prog [options] arg"
parser = OptionParser(usage)
# options for Zenoss
parser.add_option("-z", "--zenhost", dest="zenhost", help="Zenoss Server hostname", default=ZENOSS_HOST)
parser.add_option("-u", "--zenuser", dest="zenuser", help="Zenoss admin username", default=ZENOSS_USERNAME)
parser.add_option("-p", "--zenpass", dest="zenpass", help="Zenoss admin password", default=ZENOSS_PASSWORD)
parser.add_option("-e", "--evid", dest="evid", help="Zenoss Event ID", default=EVID)
parser.add_option("-V", "--zenver", dest="zenver", help="True if Zenoss version >= 4", action="store_false")
# options for Pagerduty
parser.add_option("-H", "--pdhost", dest="pdhost", help="Pagerduty hostname", default=PAGERDUTY_HOST)
parser.add_option("-T", "--pdtoken", dest="pdtoken", help="Pagerduty token", default=PAGERDUTY_TOKEN)
parser.add_option("-U", "--pduser", dest="pduser", help="Pagerduty User Key (for tracking auto updates)", default=PAGERDUTY_USER)
parser.add_option("-S", "--servicekey", dest="servicekey", help="Pagerduty Service Key", default=PAGERDUTY_SERVICEKEY)
# action to be performed
parser.add_option("-a", "--action", dest="action", help="one of [create|update]", default="update")
parser.add_option("-v", "--verbose", dest="verbose", help="Show additional output", action="store_true")
# options for zenoss interaction
(self.options, self.args) = parser.parse_args()
def run(self):
"""
control script execution
"""
if self.options.action == 'create':
self.messenger.newId(self.options.evid)
self.createPagerDutyIncident()
self.updateZenossMessages(self.options.evid)
self.updateZenossStatus(self.options.evid)
if self.options.action == 'update':
self.correlateByPagerDuty()
self.synchronize()
self.updatePagerDuty()
self.updateZenoss()
def handle_instance(self, instance, msg_type, msg):
#uninitialized instances communicating for orders
global job_list
global create_aggregator
global job_lock
id, hostname = instance.split(',')
if msg_type == "READY":
logger.info("Handling instance. Updating codebase")
logger.info(SSHCMD + hostname + SVNCMD)
logger.info(commands.getoutput(SSHCMD + hostname + SVNCMD))
try:
job_lock.acquire()
if len(job_list) == 0 and not create_aggregator:
logger.info("Unneccessary instance was spawned; terminating")
toolkit.terminateInstance(id)
pass
else:
#see which aggregator is open
job_assigned = False
for ag in aggregators:
if not job_assigned and not create_aggregator and ag['num_workers']<WORKERS_TO_AGGREGATORS:
for i in range(MAX_JOBS_QUEUE_PER_INST):
if len(job_list) > 0:
job = job_list[0]
messenger = MessageHandler(ag['hostname'], ag['port'])
msg = messenger.exchangeMessage("MASTER:JOB:" + job)
messenger.close()
# now the aggregator has the job
if msg!= "Job Ack":
logger.error("No Job Ack recv. CRITICAL ERROR!")
self.wfile.write("WORKER:%s:%s\n"
%(ag['hostname'], ag['port']))
logger.info(job + " assigned to Aggregator %s ubuntu@%s"
% (ag['id'], ag['hostname']))
logger.info("Worker %s created ubuntu@%s" % (id, hostname))
ag['num_workers'] += 1
workers[id] = ag['id']
job_list = job_list[1:]
job_assigned = True
if not job_assigned or create_aggregator:
self.wfile.write("AGGREGATOR:%s\n" % (AGGREGATOR_PORT))
logger.info("Aggregator created: %s ubuntu@%s" % (id, hostname))
ag = { }
create_aggregator = False
ag['hostname'] = hostname
ag['id'] = id
ag['port'] = AGGREGATOR_PORT
ag['num_workers'] = 0
aggregators.append(ag)
#launch an ec2 instance
toolkit.runInstance()
finally:
job_lock.release()
if msg_type == "JOB_COMPLETE":
worker_id, worker_job = msg.split(',')
logger.info("Job Complete: [AGR %s ubuntu@%s] [WORKER %s] [Job %s]" % (id, hostname, worker_id, worker_job))
if msg_type == "JOB_ASSIGN":
worker_id, worker_job = msg.split(',')
logger.info("Job Assign: [AGR %s ubuntu@%s] [WORKER %s] [Job %s]" % (id, hostname, worker_id, worker_job))
if msg_type == "DATA":
logger.info(msg)
if msg_type == "FINISH":
for ag in aggregators:
if ag['id'] == workers[id]:
ag['num_workers'] -= 1
workers[id] = None
logger.info("Command finished, terminating instance " + instance)
toolkit.terminateInstance(id)
try:
job_lock.acquire()
if toolkit.num_instances < MAX_NUM_INSTANCES and len(job_list) > 0:
toolkit.runInstance()
finally:
job_lock.release()
class ExampleBot(SlackBot) :
def __init__(self, token, debug=False):
super().__init__(token, debug)
self.users = {}
self.running = False
self.messageResponseQueue = Queue.Queue()
self.messageRequestQueue = Queue.Queue()
self.MessageHandler = None
self.debug = debug
if(self.debug) : logger.debug('ExampleBot successfully created')
# Entry method for Program
def run(self) :
self.running = True
if(self.connect()) :
# Starts thread and passes BotID
self.setUpThreads()
while(self.running) :
try:
rawInput = self.readChannels()
if(self.debug) : logger.debug(rawInput)
if (not self.isEmpty(rawInput) and self.notSelf(rawInput) and (self.botMentioned(rawInput) or self.directMessaged(rawInput)) ) :
self.messageRequestQueue.put(rawInput)
if(self.debug) : logger.info("Added message to queue: {}".format(str(rawInput)))
self.checkResponseQueue()
time.sleep(0.5)
except (KeyboardInterrupt, SystemError) :
if(self.debug) : logger.info("KeyboardInterrupt Exception Found. Killing MessageHandler Thread")
self.MessageHandler.kill()
self.running = False
else :
if(self.debug) : logger.error("Unable to connect to Slack.")
def setUpThreads(self) :
self.MessageHandler = MessageHandler(self.messageRequestQueue, self.messageResponseQueue, self.getBotID(), self.debug)
self.MessageHandler.setName("MessageHandler Thread 1")
self.MessageHandler.daemon = True
self.MessageHandler.start()
if(self.debug) : logger.info("Started thread: {}".format("MessageHandler Thread 1"))
# Check Queue to see if any messages are ready to be sent
def checkResponseQueue(self) :
if(not self.messageResponseQueue.empty()) :
response = self.messageResponseQueue.get()
if(self.debug) : logger.info("Response found, handling response: \n{}\n".format(str(response)))
self.messageResponseQueue.task_done()
self.handleResponse(response)
# Utilizes response object to response to user
def handleResponse(self, response) :
if(response['action'] == "writeToSlack") :
# TODO: Handle unsuccess result and log failures
if(self.debug) : logger.info("Writing message {} to slack channel {}".format(str(response), str(response['channel'])))
result = self.writeToSlack(response['channel'], response['response'])
elif(response['action'] == "writeToFile") :
# TODO: Make dynamic
if(self.debug) : logger.info("Writing file {} to slack channel {}".format(str(response), str(response['channel'])))
result = self.writeToFile(response['channel'], response['response'])
elif(response['action'] == "writeToSharedFile") :
if(self.debug) : logger.debug("Writing shared file {} to slack channel {}".format(str(response), str(response['channel'])))
result = self.writeToSharedFile(response['channel'], response['response'])
else :
if(self.debug) : logger.warning("Error has occured while handling response {} to channel {}".format(str(response), str(response['channel'])))
# Determines if the bot is being DMed
def directMessaged(self, rawInput) :
if('channel' in rawInput[0]):
if (not self.getGroupInfo(rawInput[0]['channel']).get('ok')) and (not self.getChannelInfo(rawInput[0]['channel']).get('ok')) :
return True
else :
return False
else :
return False
# Determines if the bot is mentioned in the raw input
def botMentioned(self, rawInput) :
if('text' in rawInput[0]) :
if(self.getBotID() in rawInput[0]['text']) : return True
else : return False
# Detects if raw input is Empty
def isEmpty(self, rawInput) :
if not rawInput : return True
else : return False
# Determines if the raw input was sent by this bot
def notSelf(self, rawInput) :
if(rawInput[0].get('user') == self.getBotID()) : return False
else : return True
class Sync():
''''''
def __init__(self, zenhost, zenuser, zenpass, pdhost, pdtoken, pduser, verbose=False):
''''''
self.zenhost = zenhost
self.zenuser = zenuser
self.zenpass = zenpass
self.pdhost = pdhost
self.pdtoken = pdtoken
self.pduser = pduser
self.logs = [] # array to hold log messages
self.verbose = verbose
self.buffersize = 100
self.statusData = {
"ack": {
"zenoss":"Acknowledged",
"pagerduty":"acknowledged",
"zenaction":"acknowledge",
"num":1
},
"close":{
"zenoss":"Closed",
"pagerduty":"resolved",
"zenaction":"close",
"num":2
},
"open":{
"zenoss":"New",
"pagerduty":"triggered",
"zenaction":"reopen",
"num":0},
}
self.zenver = self.isVersion4()
if self.zenver is True: #zenoss 4 changed "unacknowledge" action to "reopen"
self.statusData["open"]["zenaction"] = "reopen"
self.messenger = MessageHandler()
self.statusDict = {}
self.commonkeys = []
def mapCurrentStatus(self, evid, data, pd=False):
'''
map event/incident status to local dictionary
'''
ZENMAP = {'New': 'open', 'Acknowledged': 'ack', 'Closed': 'close', 'Cleared': 'close', 'Aged': 'close'}
PDMAP = {'triggered': 'open', 'acknowledged': 'ack', 'resolved': 'close'}
if pd is False:
self.statusDict[evid]['zenoss'] = data
self.statusDict[evid]["zencurrent"] = ZENMAP[data['eventStateString']]
else:
self.statusDict[evid]['pagerduty'] = data
self.statusDict[evid]["pdcurrent"] = PDMAP[data['status']]
def initZenoss(self):
'''initialize connection to Zenoss'''
self.zenoss = ZenossHandler(self.zenhost, self.zenuser, self.zenpass, self.verbose)
self.zenoss.buffersize = self.buffersize
def initPagerDuty(self):
'''initialize connection to PagerDuty'''
self.pagerduty = PagerDutyHandler(self.pdhost, self.pdtoken, self.verbose)
self.pagerduty.buffersize = self.buffersize
def initialize(self):
'''initialize connection to both Zenoss and PagerDuty'''
self.initZenoss()
self.initPagerDuty()
def isVersion4(self):
'''detect Zenoss version'''
from Products.ZenUtils.Version import Version
if Version.parse('Zenoss ' + ZENOSS_VERSION) >= Version.parse('Zenoss 4'): return True
return False
def getMaintenanceWindows(self, service):
"""
return list of maintenance windows (ongoing) for a given service
"""
if self.verbose is True: self.writelog("Finding PagerDuty Maintenance Windows for %s" % service['id'])
output = []
windows = self.pagerduty.getMaintenanceWindows()["maintenance_windows"]
for w in windows:
services = w["services"]
for s in services:
if s["id"] == service["id"]:
beg = self.messenger.getLocalTime(w["start_time"])
fin = self.messenger.getLocalTime(w["end_time"])
start = self.messenger.getTimestamp(beg)
end = self.messenger.getTimestamp(fin)
now = time.time()
if start <= now and now <=end: output.append(w)
return output
def getZenEvent(self, evid, history=False):
''''''
output = self.zenoss.getEventDetails(evid)
if 'result' not in output.keys(): return None
if 'event' not in output['result'].keys(): return None
if len(output['result']['event']) != 1: return None
return output["result"]["event"][0]
def getPDIncident(self, evid):
''''''
incident = None
output = self.pagerduty.getIncidentByKey(evid)
#if 'incidents' not in output.keys(): return incident
if 'incidents' in output.keys():
count = len(output['incidents'])
# if more than one, take the most recent
if count >= 1:
max = 0
for i in output['incidents']:
pagerdutytime = i["last_status_change_on"]
pagerdutytime_local = self.messenger.getPagerDutyTime(pagerdutytime)
pagerdutytimestamp = self.messenger.getTimestamp(pagerdutytime_local)
if pagerdutytimestamp > max:
incident = i
max = pagerdutytimestamp
if incident is not None: self.addStatusDictEntry(evid, incident, True)
return incident
def getZenEventDict(self, evid):
'''
standardize zenoss 3.x vs. 4.x output
'''
if self.verbose is True: self.writelog("Finding Zenoss event details for %s" % evid)
# map of string to numeric event status
STATUSMAP3X = { 0: 'New', 1: 'Acknowledged', 2: 'Closed', }
STATUSMAP4X = { 'New': 0, 'Acknowledged': 1, 'Closed': 2, 'Cleared': 2, 'Aged': 2, }
data = {'history': False}
# try getting from current console
event = self.getZenEvent(evid)
if self.zenver == False: # was not pulled from history
# if nothing, try looking in history
if event is None:
event = self.getZenEvent(evid, True)
data['eventState'] = 2 # setting to close since this is in history
data['history'] = True # was pulled from history
if event is None: return None
details = event["properties"]
for d in details: data[str(d["key"])] = str(d["value"])
data['eventStateInt'] = data['eventState']
data['eventStateString'] = STATUSMAP3X[data['eventState']]
data['log'] = output['log']
else: # zenoss 4.x output
if event is None: return None
data.update(event)
data['history'] = False
data['eventStateString'] = data['eventState']
data['eventStateInt'] = STATUSMAP4X[data['eventState']]
self.addStatusDictEntry(evid, data)
return data
def addStatusDictEntry(self, evid, data, pd=False):
'''
add entry to status dict for this evid
'''
if evid not in self.statusDict.keys():
self.statusDict[evid] = {'target': None, 'zencurrent': None, 'pdcurrent': None, 'zenoss' : None, 'pagerduty': None,}
self.mapCurrentStatus(evid, data, pd)
def createIncidentDetails(self, evid, servicekey):
"""
Retrieve event detail from Zenoss and format it
for PagerDuty incident creation
"""
if self.verbose is True: self.writelog("Getting incident details")
data = self.getZenEventDict(evid)
info = {
"service_key": servicekey,
"incident_key": evid,
"details" : self.statusDict[evid]['zenoss'],
"description" : None,
}
self.statusDict[evid]["target"] = "open"
info["details"] = data
info["description" ] = ' | '.join([str(data['device']), str(data['component']), str(data['summary'])])
return info
def createPagerDutyIncident(self, evid, servicekey):
"""
1) check whether the destination service is in a maintenance window.
a) If so, ack the local alert in zenoss
b) and add a "in maintenance window" log message to the Zenoss event details.
2) if it is not in maintenance, proceed with the event submission. as usual
a) send event to pagerduty
b) update Zenoss console with submit status info
c) update Zenoss Event with incident details (incident URL, service URL, oncall)
3) check issues in Zenoss w/pagerduty incidents:
a) if acked in Zenoss, ack in PagerDuty
b) if closed in Zenoss, resolve in PagerDuty
"""
if self.verbose is True: self.writelog("Creating PagerDuty Incident")
info = self.createIncidentDetails(evid, servicekey)
# first ensure that event is open in zenoss
if self.statusDict[evid]["zencurrent"] == 'open':
# first find the appropriate PD service definition
if self.verbose is True: self.writelog("looking for service using key %s" % servicekey)
service = self.pagerduty.findService(servicekey)
if service is not None:
if self.verbose is True: self.writelog("found service using key %s" % servicekey)
# in maintenance, so ack the zenoss alert but note the window detail
if self.pagerduty.inMaintenance(service) is True:
if self.verbose is True: self.writelog("service in maintenance using key %s" % servicekey)
self.statusDict[evid]["target"] = "ack"
mws = self.getMaintenanceWindows(service)
for mw in mws: self.messenger.serviceInMaintenance(evid, "Acknowledged", service, mw, self.pagerduty.weburl)
# disabled, so leave event unacked in Zenoss, but that service is disabled
elif self.pagerduty.isDisabled(service) is True:
if self.verbose is True: self.writelog("service disabled using key %s" % servicekey)
self.messenger.serviceIsDisabled(evid, "No Incident created", service, self.pagerduty.weburl)
# assuming service is enabled, create PD incident, note output in zenoss event console.
else:
if self.verbose is True: self.writelog("Creating incident for %s using key %s" % (servicekey, evid))
output = self.pagerduty.manageIncident(info, "trigger")
if 'errors' in output.keys(): errors = output['errors']
else: errors = None
self.messenger.serviceIncidentCreated(evid, service, self.pagerduty.weburl, errors)
else:
if self.verbose is True: self.writelog("No service found using key %s" % servicekey)
self.messenger.serviceNotFound(evid, servicekey)
else: # if not open in zenoss, set the target to whatever is current
if self.verbose is True: self.writelog("Event id %s not open in Zenoss" % evid)
self.statusDict[evid]["target"] = self.statusDict[evid]["zencurrent"]
def getPDKeys(self, open=False):
'''get incident ids for last N incidents (buffersize)'''
pdkeys = []
# get PagerDuty incidents
incidents = self.pagerduty.getIncidentList(open=open)
if 'incidents' not in incidents.keys(): return pdkeys
# make list of event ids for both
for i in incidents["incidents"]: pdkeys.append(str(i['incident_key']))
return pdkeys
def getZenKeys(self, open=False):
'''get the evids for the last N events (buffersize)'''
zenkeys = []
try: events = self.zenoss.getEvents(open=open)["events"]
except: events = []
# zenoss 3.x compat
if self.zenver is False:
try: events += self.zenoss.getEvents(history=True, open=open)["events"]
except: pass
# make list of event ids for both
for e in events: zenkeys.append(str(e['evid']))
self.writelog('Found %s open Zenoss events' % len(zenkeys))
return zenkeys
def correlate(self):
'''
1) get list of open PD incidents
2) correlate with open or closed Zenoss events
3) get list of open Zenoss incidents not in 2)
4) correlate with open or closed PD incidents
build dictionary of Zenoss events matched to PagerDuty incidents
1) get list of zenoss events
2) get list of pagerduty incidents based on date range of zenoss events
3) match them by evid - incident_key
'''
if self.verbose is True: self.writelog("Finding correlated Zenoss events and PagerDuty incidents")
# look at all open PD incidents
pdkeys = self.getPDKeys(True)
# find the matching PD/zenoss data
for id in pdkeys:
zdata = self.getZenEventDict(id) # get zenoss data
if zdata is None: continue
pddata = self.getPDIncident(id)
if pddata is None: continue
if id not in self.commonkeys: self.commonkeys.append(id)
# look at all open zenoss events
zenkeys = self.getZenKeys(True)
# find matching PD data
for id in zenkeys:
# no reason to do work twice
if id in self.commonkeys:
self.writelog('skipping already processed id: %s' % id)
continue
pddata = self.getPDIncident(id)
if pddata is None: continue
zdata = self.getZenEventDict(id) # get zenoss data
if zdata is None: continue
if id not in self.commonkeys: self.commonkeys.append(id)
def synchronize(self):
"""
update pagerduty based on Zenoss event console activities (acknowledge, resolve, etc)
4) if acked in one, ack in the other
5) if closed in one, close in the other
"""
self.correlate()
if self.verbose is True: self.writelog("Synchronizing correlated Zenoss events and PagerDuty incidents")
for id, data in self.statusDict.items():
if self.verbose is True: self.writelog('Synchronizing %s' % self.eventInfoMsg(data))
# map current states of each to statusDict
self.evalStatus(data)
# determine which was updated last
self.findNewest(data)
# update pd and zenoss
self.updateStatus(data)
# gather/format any log messages in pagerduty
self.buildMessages(data)
# update Zenoss event logs
self.updateZenossMessages(data)
def evalStatus(self, data):
'''update the status dictionary entry'''
if self.verbose is True: self.writelog("Evaluating status data: %s" % self.eventInfoMsg(data))
for status, tdata in self.statusData.items():
if tdata["zenoss"] == data['zenoss']['eventStateString']: data['zencurrent'] = status
if tdata["pagerduty"] == data['pagerduty']['status']: data['pdcurrent'] = status
def buildMessages(self, data):
''' build the log messages that will show in the Zenoss console'''
if self.verbose is True: self.writelog("Building messages for %s" % self.eventInfoMsg(data))
# gather/format any log messages in pagerduty
evid = str(data['zenoss']['evid'])
pdid = str(data['pagerduty']['id'])
# basic incident creation message
self.messenger.incidentCreated(evid, data['pagerduty'])
# any others from pagerduty
logs = self.pagerduty.getIncidentDetail(pdid)["log_entries"]
if self.verbose is True: self.writelog("Found %s incident logs for %s" % (len(logs), self.eventInfoMsg(data)))
self.messenger.incidentLogs(evid, logs, self.pagerduty.weburl)
def findNewest(self, data):
''''''
if self.verbose is True: self.writelog("finding newest source for %s" % self.eventInfoMsg(data))
# event was pulled from history
if data['zenoss']['history'] is True: zenosstime = data['zenoss']['deletedTime']
# otherwise just the last time it changed
else: zenosstime = data['zenoss']['stateChange']
pagerdutytime = data['pagerduty']["last_status_change_on"]
# both are converted to local time
zenosstime_local = self.messenger.getZenossTime(zenosstime)
pagerdutytime_local = self.messenger.getPagerDutyTime(pagerdutytime)
if self.verbose is True: self.writelog("%s updated in Zenoss at %s and PagerDuty at %s" % (self.eventInfoMsg(data), zenosstime_local, pagerdutytime_local))
# then both are converted to integers
zenosstimestamp = self.messenger.getTimestamp(zenosstime_local)
pagerdutytimestamp = self.messenger.getTimestamp(pagerdutytime_local)
if zenosstimestamp < pagerdutytimestamp: # pagerduty updated last
data['target'] = data['pdcurrent']
newest = "PAGERDUTY"
else:
data['target'] = data['zencurrent']
newest = "ZENOSS"
if self.verbose is True: self.writelog("%s last updated in %s with %s status" % (self.eventInfoMsg(data), newest, data['target'].upper()))
def updateStatus(self, data):
'''
perform updates on Zenoss and Pagerduty
'''
if self.verbose is True: self.writelog("Updating status for %s" % self.eventInfoMsg(data))
# zenoss event id
evid = str(data['zenoss']['evid'])
# pagerduty incident id
pdid = data['pagerduty']['id']
# zenoss current state
zencurrent = data['zencurrent']
# pagerduty current state
pdcurrent = data['pdcurrent']
# target state
target = data['target']
# check Zenoss status against target
if zencurrent != target:
# ze:enoss 3.x does not change the status of a closed event
if target == "close" and data['zenoss']['history'] is True:
if self.verbose is True: self.writelog("Cannot update historical event %s" % self.eventInfoMsg(data))
else:
if self.verbose is True: self.writelog("Changing Zenoss event %s status from %s to %s" % (self.eventInfoMsg(data), zencurrent.upper(), target.upper()))
self.zenoss.manageEventStatus([evid], self.statusData[target]['zenaction'])
# check PagerDuty status against target
if pdcurrent != target:
if self.verbose is True: self.writelog("Changing PagerDuty incident %s status from %s to %s" % (self.eventInfoMsg(data), pdcurrent.upper(), target.upper()))
try:
pdData = {"id": pdid, "status": self.statusData[target]["pagerduty"]}
# test individually
update = {"incidents": [pdData], "requester_id": self.pduser}
self.pagerduty.updateStatus(update)
except: self.writelog("%s update pagerduty failed changing %s to %s" % (self.eventInfoMsg(data), pdcurrent.upper(), target.upper()))
def updateZenossStatus(self, data):
"""
update Zenoss event status if target is not current
"""
if self.verbose is True: self.writelog("Checking Zenoss status for %s" % self.eventInfoMsg(data))
# zenoss event id
evid = str(data['zenoss']['evid'])
current = data["zencurrent"]
target = data["target"]
if current != target:
if self.verbose is True: self.writelog("Changing %s status to %s" % (self.eventInfoMsg(data), target.upper()))
self.zenoss.manageEventStatus([evid], self.statusData[target]['zenaction'])
def updateZenossMessages(self, data):
"""
update Zenoss event console with messages if they don't exist
"""
if self.verbose is True: self.writelog("Checking Zenoss messages for %s" % self.eventInfoMsg(data))
# zenoss event id
evid = str(data['zenoss']['evid'])
# get list of current event log messages
zlogs = []
for e in data['zenoss']['log']: zlogs.append(str(e[-1]))
for msg in self.messenger.messages[evid]:
#self.writelog('examining message: "%s"' % msg)
if msg not in zlogs:
if self.verbose is True: self.writelog('Adding message to %s: "%s"' % (self.eventInfoMsg(data), msg))
self.zenoss.addEventMessage(evid, msg)
def updateCreatedIssue(self):
''' update zenoss status and messages for new issue'''
if self.verbose is True: self.writelog("updating created issue")
for id, data in self.statusDict.items():
self.updateZenossStatus(data)
self.updateZenossMessages(data)
def writelog(self,msg):
''' print log message and append to list for log handling'''
self.logs.append(msg)
print msg
def eventInfoMsg(self, data):
'''convenience for log messages'''
try: return '%s (%s) "%s"' % (str(data['zenoss']['evid']), data['pagerduty']['id'], data['zenoss']['summary'])
except: return '%s "%s"' % (str(data['zenoss']['evid']), data['zenoss']['summary'])
def handle_instance(self, instance, msg_type, msg):
#uninitialized instances communicating for orders
global job_list
global create_aggregator
global job_lock
id, hostname = instance.split(',')
if msg_type == "READY":
logger.info("Handling instance. Updating codebase")
logger.info(SSHCMD + hostname + SVNCMD)
logger.info(commands.getoutput(SSHCMD + hostname + SVNCMD))
try:
job_lock.acquire()
if len(job_list) == 0 and not create_aggregator:
logger.info(
"Unneccessary instance was spawned; terminating")
toolkit.terminateInstance(id)
pass
else:
#see which aggregator is open
job_assigned = False
for ag in aggregators:
if not job_assigned and not create_aggregator and ag[
'num_workers'] < WORKERS_TO_AGGREGATORS:
for i in range(MAX_JOBS_QUEUE_PER_INST):
if len(job_list) > 0:
job = job_list[0]
messenger = MessageHandler(
ag['hostname'], ag['port'])
msg = messenger.exchangeMessage(
"MASTER:JOB:" + job)
messenger.close()
# now the aggregator has the job
if msg != "Job Ack":
logger.error(
"No Job Ack recv. CRITICAL ERROR!")
self.wfile.write(
"WORKER:%s:%s\n" %
(ag['hostname'], ag['port']))
logger.info(
job +
" assigned to Aggregator %s ubuntu@%s"
% (ag['id'], ag['hostname']))
logger.info("Worker %s created ubuntu@%s" %
(id, hostname))
ag['num_workers'] += 1
workers[id] = ag['id']
job_list = job_list[1:]
job_assigned = True
if not job_assigned or create_aggregator:
self.wfile.write("AGGREGATOR:%s\n" % (AGGREGATOR_PORT))
logger.info("Aggregator created: %s ubuntu@%s" %
(id, hostname))
ag = {}
create_aggregator = False
ag['hostname'] = hostname
ag['id'] = id
ag['port'] = AGGREGATOR_PORT
ag['num_workers'] = 0
aggregators.append(ag)
#launch an ec2 instance
toolkit.runInstance()
finally:
job_lock.release()
if msg_type == "JOB_COMPLETE":
worker_id, worker_job = msg.split(',')
logger.info(
"Job Complete: [AGR %s ubuntu@%s] [WORKER %s] [Job %s]" %
(id, hostname, worker_id, worker_job))
if msg_type == "JOB_ASSIGN":
worker_id, worker_job = msg.split(',')
logger.info(
"Job Assign: [AGR %s ubuntu@%s] [WORKER %s] [Job %s]" %
(id, hostname, worker_id, worker_job))
if msg_type == "DATA":
logger.info(msg)
if msg_type == "FINISH":
for ag in aggregators:
if ag['id'] == workers[id]:
ag['num_workers'] -= 1
workers[id] = None
logger.info("Command finished, terminating instance " +
instance)
toolkit.terminateInstance(id)
try:
job_lock.acquire()
if toolkit.num_instances < MAX_NUM_INSTANCES and len(
job_list) > 0:
toolkit.runInstance()
finally:
job_lock.release()
def handle_instance(self, instance, msg_type, msg):
global jobs
global job_lock
if msg_type == "READY":
try:
job_lock.acquire()
if len(jobs) == 0:
logger.info("No jobs to assign! Terminating %s" %
(INSTANCE_ID))
self.job = "TERMINATE"
else:
self.job = jobs[0] + "\n"
jobs = jobs[1:]
self.wfile.write(self.job)
messenger = MessageHandler(MASTER_IP, MASTER_PORT)
messenger.sendMessage(
"%s,%s:JOB_ASSIGN:%s,%s" %
(INSTANCE_ID, PUBLIC_HOSTNAME, instance, self.job))
messenger.close()
logger.info(self.job)
finally:
job_lock.release()
if msg_type == "DATA":
logger.info(msg)
if msg_type == "FINISH":
logger.info("Job complete: " + self.job)
messenger = MessageHandler(MASTER_IP, MASTER_PORT)
messenger.sendMessage(
"%s,%s:JOB_COMPLETE:%s,%s" %
(INSTANCE_ID, PUBLIC_HOSTNAME, instance, self.job))
messenger.close()
# Connect to twitch's irc server
irc = ircConnection()
s = socket.socket()
s.connect((irc.HOST, irc.PORT))
s.send(bytes("PASS " + irc.PASS + "\r\n", "UTF-8"))
s.send(bytes("NICK " + irc.NICK + "\r\n", "UTF-8"))
s.send(bytes("JOIN #" + irc.CHANNEL + "\r\n", "UTF-8"))
s.send(bytes("CAP REQ :twitch.tv/membership\r\n", "UTF-8"))
s.send(bytes("CAP REQ :twitch.tv/tags\r\n", "UTF-8"))
s.send(bytes("CAP REQ :twitch.tv/commands\r\n", "UTF-8"))
stream = Stream()
stream_thread = threading.Thread(target=stream.get_info, args=(channel_name,))
stream_thread.daemon = True
stream_thread.start()
mh = MessageHandler(s, irc, stream)
while True:
for rawLine in str(s.recv(524288).decode("utf8")).split("\r\n"):
server_response = str(rawLine.encode("utf8"))
if server_response.startswith("b'PING :tmi.twitch.tv"):
s.send(bytes("PONG :tmi.twitch.tv \r\n", "UTF-8"))
continue
mh.message_handler(server_response)
import SocketServer, commands, socket, time, sys
from subprocess import Popen, PIPE, STDOUT
from MessageHandler import MessageHandler
HOST = sys.argv[1]
PORT = int(sys.argv[2])
INSTANCE_ID = commands.getoutput("wget -q -O - http://169.254.169.254/latest/meta-data/instance-id")
messenger = MessageHandler(HOST, PORT)
keepRunning = True
while keepRunning:
data = INSTANCE_ID + ":READY"
msg = messenger.exchangeMessage(data)
print msg
if (msg == "TERMINATE"):
keepRunning = False
msg = "echo 'I am terminating'"
process = Popen(msg, stdout=PIPE, stdin=PIPE, stderr=STDOUT, shell=True)
data = ""
while process.poll() is None:
data += process.stdout.read()
msgs = data.split('\n')
for m in msgs[0:-1]:
print m
messenger.sendMessage(INSTANCE_ID+":DATA:"+m)
data = msgs[-1]
msg = messenger.sendMessage(INSTANCE_ID+":DATA:"+data)
messenger.sendMessage(INSTANCE_ID+":FINISH")
return map(lambda x: x*step, range(int(start*1./step),int(end*1./step)))
if __name__ == "__main__":
param_file = sys.argv[ARG_PARAM_FILE]
eval_mode = sys.argv[ARG_EVAL_MODE]
min_twist = float(sys.argv[ARG_MIN_TWIST])
iter_twist = float(sys.argv[ARG_ITER_TWIST])
max_twist = float(sys.argv[ARG_MAX_TWIST])
min_grav = float(sys.argv[ARG_MIN_GRAV])
iter_grav = float(sys.argv[ARG_ITER_GRAV])
max_grav = float(sys.argv[ARG_MAX_GRAV])
print param_file, min_twist, iter_twist, max_twist, min_grav, iter_grav, max_grav
messenger = MessageHandler(HOST, PORT)
i=0
for twist in frange(min_twist, max_twist+20, 20):
BINARY_CMD = ("%s %s %s %s %s %s %s %s %s;" %
(BINARY, param_file, eval_mode, twist, iter_twist, min(twist+20-iter_twist/2, max_twist), min_grav, iter_grav, max_grav))
job = "%s %s %s" % (COMPILE_CMD, BINARY_CMD, SCP_CMD)
print job
data = "USER:JOB:" + job
messenger.sendMessage(data)
time.sleep(0.1)
i += 1
messenger.close()
print i
def setUpThreads(self) :
self.MessageHandler = MessageHandler(self.messageRequestQueue, self.messageResponseQueue, self.getBotID(), self.debug)
self.MessageHandler.setName("MessageHandler Thread 1")
self.MessageHandler.daemon = True
self.MessageHandler.start()
if(self.debug) : logger.info("Started thread: {}".format("MessageHandler Thread 1"))
from subprocess import Popen, PIPE, STDOUT
from logger import EC2Logger
from MessageHandler import MessageHandler
HOST = "128.32.37.215"
PORT = 10000
INSTANCE_ID = commands.getoutput(
"wget -q -O - http://169.254.169.254/latest/meta-data/instance-id")
PUBLIC_HOSTNAME = commands.getoutput(
"wget -q -O - http://169.254.169.254/latest/meta-data/public-hostname")
AGGREGATOR_CMD = "python /home/ubuntu/code/trunk/surgical/ec2/threaded_aggregator.py "
WORKER_CMD = "python /home/ubuntu/code/trunk/surgical/ec2/worker.py "
logger = EC2Logger("CFG-" + INSTANCE_ID)
messenger = MessageHandler(HOST, PORT)
data = INSTANCE_ID + "," + PUBLIC_HOSTNAME
logger.info(data + ":READY")
msg = messenger.exchangeMessage(data + ":READY")
messenger.close()
msg = msg.split(":")
if msg[0] == "AGGREGATOR":
logger.info("Starting Aggregator Service")
process = Popen("%s %s" % (AGGREGATOR_CMD, msg[1]), shell=True)
elif msg[0] == "WORKER":
logger.info("Starting Worker Service")
process = Popen("%s %s %s" % (WORKER_CMD, msg[1], msg[2]), shell=True)
while process.poll() is None:
time.sleep(15)
from drivertest import Elevator
from Communicator import Communicator
from MessageHandler import MessageHandler
from Controller import Controller
from Message import Message
from Peer import Peer
#Elevator()
messageHandler = MessageHandler()
communicator = Communicator(messageHandler)
controller = Controller(communicator)
messageHandler.setController(controller)
print "main"
message = Message("newOrder")
print message
peer = Peer("129.241.187.145")
#communicator.broadcast(message)
#communicator.sendToOne(peer, message)
print "main end"
class HypeBot(ClientXMPP):
def __init__(self, config_file):
# setup configuration
self.config = ConfigParser.RawConfigParser()
self.config.read(config_file)
self.user_nickname = self.config.get(CONFIG_AUTH, 'user_nickname')
self.user_jid = self.config.get(CONFIG_AUTH, 'user_jid')
self.user_pwd = self.config.get(CONFIG_AUTH, 'user_pwd')
self.user_api_token = self.config.get(CONFIG_AUTH, 'user_api_token')
# setup xmpp client
ClientXMPP.__init__(self, self.user_jid, self.user_pwd)
self.add_event_handler("session_start", self.session_start)
self.add_event_handler("message", self.message)
# register plugins for additional functionality
self.register_plugin('xep_0030') # Service Discovery
self.register_plugin('xep_0004') # Data Forms
self.register_plugin('xep_0045') # MUC
self.register_plugin('xep_0060') # PubSub
self.register_plugin('xep_0199') # Ping
# Setup message handler
self.msg_handler = MessageHandler(self)
# Setup HypChat api client
self.hc = HypChat(self.user_api_token)
# get jid to room map
self.jid_to_room = dict()
# Join rooms on startup
startup_rooms = self.config.get(CONFIG_GENERAL, 'startup_rooms_to_join').split(',')
for room in startup_rooms:
self.join_room_by_name(room)
print('Bot initialized')
def session_start(self, event):
self.get_roster()
self.send_presence(ppriority=0)
# enable keepalive, times are in seconds
self.plugin['xep_0199'].enable_keepalive(interval=30, timeout=30)
def populate_jid_to_room(self, room_name):
room = self.hc.get_room(room_name)
self.jid_to_room[room['xmpp_jid']] = room
return room
# rooms = self.hc.rooms()
# room_list = list(rooms.contents())
# for room in room_list:
# room_self = room.self()
# self.jid_to_room[room_self['xmpp_jid']] = room
# Join a hipchat room
def join_room(self, room_jid):
self.plugin['xep_0045'].joinMUC(room_jid, self.user_nickname, maxhistory=None, wait=True)
def join_room_by_name(self, room_name):
# Populate a map from jid to room
# and return the room at the same time
# this should help with rate-limiting on api calls
print("Joining room: " + room_name)
room_to_join = self.populate_jid_to_room(room_name)
if room_to_join is None:
return False
self.join_room(room_to_join['xmpp_jid'])
return True
def reply_room_name(self, room_name, body):
room_to_reply = self.hc.get_room(room_name)
if room_to_reply is None:
return False
self.send_message(mto=room_name, mbody=body, mtype='groupchat')
return True
def reply_room(self, msg, body):
print(msg['from'].bare)
self.send_message(mto=msg['from'].bare, mbody=body, mtype='groupchat')
def message(self, msg):
self.msg_handler.handle(msg)
def notify_room_html(self, text, jid):
self.jid_to_room[jid].notification(text, format='html')
from __future__ import division, print_function, unicode_literals, absolute_import
import warnings
warnings.simplefilter('default', DeprecationWarning)
import os, sys
import numpy as np
import xml.etree.ElementTree as ET
frameworkDir = os.path.normpath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir,
os.pardir, 'framework'))
sys.path.append(frameworkDir)
from utils import xmlUtils, utils
from MessageHandler import MessageHandler
mh = MessageHandler()
mh.initialize({})
print(xmlUtils)
results = {"pass": 0, "fail": 0}
#type comparison
elemType = type(ET.Element('dummy'))
treeType = type(ET.ElementTree())
#cleanup utilities
toRemove = []
def checkAnswer(comment, value, expected, tol=1e-10, updateResults=True):
"""
This method is aimed to compare two floats given a certain tolerance
class Main():
"""
Either:
1) Create Pagerduty Incident based on new Zenoss event
2) Synchronize Zenoss and Pagerduty Events and Incidents
"""
def __init__(self):
"""
"""
self.getOptions()
#self.options.verbose = True
self.zenoss = ZenossHandler(self.options.zenhost, self.options.zenuser,
self.options.zenpass, self.options.verbose)
self.pagerduty = PagerDutyHandler(self.options.pdhost,
self.options.pdtoken,
self.options.verbose)
# dictionary of corresponding event/incident states
self.statusData = {
"ack": {
"zenoss": "Acknowledged",
"pagerduty": "acknowledged",
"zenaction": "acknowledge",
"num": 1
},
"close": {
"zenoss": "Closed",
"pagerduty": "resolved",
"zenaction": "close",
"num": 2
},
"open": {
"zenoss": "New",
"pagerduty": "triggered",
"zenaction": "unacknowledge",
"num": 0
},
}
if self.options.zenver == True:
self.statusData["open"]["zenaction"] = "reopen"
self.messenger = MessageHandler()
self.statusDict = {}
def lastDisabled(self, service):
"""
return PagerDuty newest disabled entry
"""
message = ""
details = self.pagerduty.getServiceLog(service["id"])["log_entries"]
last = 0
lastentry = None
for d in details:
if d["maintenance_window"]["type"] == "disable":
starttime = self.messenger.getLocalTime(
d["maintenance_window"]["time"])
start = self.messenger.getTimestamp(starttime)
if start > last:
lastentry = d
last = start
return lastentry
def getMaintenanceWindows(self, service):
"""
return list of maintenance windows (ongoing) for a given service
"""
output = []
windows = self.pagerduty.getMaintenanceWindows()["maintenance_windows"]
for w in windows:
services = w["services"]
for s in services:
if s["id"] == service["id"]:
beg = self.messenger.getLocalTime(w["start_time"])
fin = self.messenger.getLocalTime(w["end_time"])
start = self.messenger.getTimestamp(beg)
end = self.messenger.getTimestamp(fin)
now = time.time()
if start <= now and now <= end:
output.append(w)
return output
def createIncidentDetails(self, evid):
"""
Retrieve event detail from Zenoss and format it
for PagerDuty incident creation
"""
self.incidentData = {
"service_key": self.options.servicekey,
"incident_key": self.options.evid
}
details = self.zenoss.getEventDetails(
evid)["result"]["event"][0]["properties"]
self.statusDict[evid] = {}
self.statusDict[evid]["target"] = "open"
data = {}
status = 0
dev = None
comp = None
summ = None
for d in details:
k = d["key"]
v = d["value"]
data[k] = v
if k == "device": dev = v
if k == "component": comp = v
if k == "summary": summ = v
if k == "eventState": status = v
self.incidentData["details"] = data
self.incidentData["description"] = "%s | %s | %s" % (dev, comp, summ)
if status == 0:
self.statusDict[evid]["current"] = "open"
elif status == 1:
self.statusDict[evid]["current"] = "ack"
elif status == 2:
self.statusDict[evid]["current"] = "close"
return status
def createPagerDutyIncident(self):
"""
1) check whether the destination service is in a maintenance window.
a) If so, ack the local alert in zenoss
b) and add a "in maintenance window" log message to the Zenoss event details.
2) if it is not in maintenance, proceed with the event submission. as usual
a) send event to pagerduty
b) update Zenoss console with submit status info
c) update Zenoss Event with incident details (incident URL, service URL, oncall)
3) check issues in Zenoss w/pagerduty incidents:
a) if acked in Zenoss, ack in PagerDuty
b) if closed in Zenoss, resolve in PagerDuty
"""
status = self.createIncidentDetails(self.options.evid)
if status == 0:
# first find the appropriate PD service definition
service = self.pagerduty.findService(self.options.servicekey)
if service:
# in maintenance, so ack the zenoss alert but note the window detail
if self.pagerduty.inMaintenance(service) == True:
self.statusDict[self.options.evid]["target"] = "ack"
mws = self.getMaintenanceWindows(service)
for mw in mws:
self.messenger.serviceInMaintenance(
self.options.evid, "Acknowledged", service, mw,
self.pagerduty.weburl)
# disabled, so leave event unacked in Zenoss, but that service is disabled
elif self.pagerduty.isDisabled(service) == True:
self.messenger.serviceIsDisabled(
self.options.evid, "No Incident created", service,
self.lastDisabled(service), self.pagerduty.weburl)
# assuming service is enabled, create PD incident, note output in zenoss event console.
else:
output = self.pagerduty.manageIncident(
self.incidentData, "trigger")
try:
self.messenger.serviceIncidentCreated(
self.options.evid, service, self.pagerduty.weburl,
output["errors"])
except KeyError:
self.messenger.serviceIncidentCreated(
self.options.evid, service, self.pagerduty.weburl)
else:
self.messenger.serviceNotFound(self.options.evid,
self.options.servicekey)
else:
self.statusDict[self.options.evid]["target"] = self.statusDict[
self.options.evid]["current"]
def correlateByZenoss(self):
"""
build dictionary of Zenoss events matched to PagerDuty incidents
1) get list of zenoss events
2) get list of pagerduty incidents based on date range of zenoss events
3) match them by evid - incident_key
"""
if self.options.verbose == True:
print "CORRELATING ZENOSS EVENTS AND PAGERDUTY INCIDENTS"
#incidents = self.pagerduty.getIncidentList()["incidents"]
events = self.zenoss.getEvents()["events"]
self.pairs = []
for e in events:
data = {"id": e["evid"], 'pagerduty': None, 'zenoss': e}
try:
data["pagerduty"] = self.pagerduty.getIncidentByKey(e["evid"])
except:
pass
self.pairs.append(data)
def correlateByPagerDuty(self):
"""
build dictionary of Zenoss events matched to PagerDuty incidents
1) get list of zenoss events
2) get list of pagerduty incidents based on date range of zenoss events
3) match them by evid - incident_key
"""
if self.options.verbose == True:
print "CORRELATING ZENOSS EVENTS AND PAGERDUTY INCIDENTS"
incidents = self.pagerduty.getIncidentList()["incidents"]
events = self.zenoss.getEvents()["events"]
self.pairs = []
for i in incidents:
data = {"id": i["incident_key"], 'pagerduty': i, 'zenoss': None}
for e in events:
if i["incident_key"] == e["evid"]:
data['zenoss'] = e
self.pairs.append(data)
def synchronize(self):
"""
update pagerduty based on Zenoss event console activities (acknowledge, resolve, etc)
4) if acked in one, ack in the other
5) if closed in one, close in the other
"""
if self.options.verbose == True:
print "SYNCHRONIZING ZENOSS AND PAGERDUTY EVENT/INCIDENT STATES"
for p in self.pairs:
id = p["id"]
zen = p["zenoss"]
pd = p["pagerduty"]
self.messenger.newId(id)
self.statusDict[id] = {}
# if pagerduty event is open, but not listed in zenoss, then close it (for Zenoss 3.x, to avoid querying event history)
if zen == None:
#if self.options.zenver == False:
self.statusDict[id]["target"] = "close"
else:
# set up console messages
self.messenger.incidentCreated(id, pd)
#self.messenger.incidentAssigned(id, pd)
details = self.pagerduty.getIncidentDetail(
pd["id"])["log_entries"]
self.messenger.incidentLogs(id, details, self.pagerduty.weburl)
for k, v in self.statusData.items():
if v["zenoss"] == zen["eventState"]:
self.statusDict[id]["current"] = k
# determine whether Zenoss or Pagerduty has authority
self.compare(zen, pd)
def compare(self, event, incident):
"""
determine target state based on event vs. incident last updated
"""
if self.options.verbose == True:
print "COMPARING EVENT AND INCIDENT FOR ID:%s" % event["evid"]
evupdated = None
evdetails = details = self.zenoss.getEventDetails(
event["evid"])["result"]["event"][0]["properties"]
for e in evdetails:
k = e["key"]
v = e["value"]
if k == "stateChange": eupdated = v
iupdated = incident["last_status_change_on"]
zt = self.messenger.getZenossTime(eupdated)
pt = self.messenger.getUTCTime(iupdated)
if self.options.verbose == True:
print "COMPARING zenoss time %s and pagerduty time %s" % (eupdated,
iupdated)
print "COMPARING zenoss time %s and pagerduty time %s" % (zt, pt)
zentime = self.messenger.getTimestamp(zt)
pdtime = self.messenger.getTimestamp(pt)
zenstate = event['eventState']
pdstate = incident['status']
if self.options.verbose == True:
print "EVENT ID %s IS %s IN ZENOSS, %s IN PAGERDUTY" % (
event["evid"], zenstate, pdstate)
zentarget = None
pdtarget = None
for k, v in self.statusData.items():
if v["zenoss"] == zenstate:
zentarget = k
if v["pagerduty"] == pdstate:
pdtarget = k
if zentime >= pdtime:
if self.options.verbose == True:
print "ZENOSS %s NEWER THAN PAGERDUTY %s" % (zenstate, pdstate)
self.statusDict[event["evid"]]["target"] = zentarget
else:
if self.options.verbose == True:
print "PAGERDUTY %s NEWER THAN ZENOSS %s" % (pdstate, zenstate)
self.statusDict[event["evid"]]["target"] = pdtarget
def getIncidentLogs(self, id, evid):
details = self.pagerduty.getIncidentDetail(id)["log_entries"]
self.messenger.incidentLogs(evid, details, self.pagerduty.weburl)
def updatePagerDuty(self):
"""
update PagerDuty incident status
"""
if self.options.verbose == True:
print "UPDATING PAGERDUTY"
updates = {"incidents": [], "requester_id": self.options.pduser}
for p in self.pairs:
id = p["id"]
pd = p["pagerduty"]
target = self.statusDict[id]["target"]
if pd["status"] != self.statusData[target]["pagerduty"]:
if self.options.verbose == True:
print "INCIDENT %s IS %s; SHOULD BE %s" % (
pd["id"], pd["status"],
self.statusData[target]["pagerduty"])
pdData = {
"id": pd["id"],
"status": self.statusData[target]["pagerduty"]
}
updates["incidents"].append(pdData)
if len(updates["incidents"]) > 0:
self.pagerduty.updateStatus(updates)
def updateZenoss(self):
"""
update Zenoss console
"""
if self.options.verbose == True:
print "UPDATING ZENOSS"
for p in self.pairs:
id = p["id"]
zen = p["zenoss"]
if p["zenoss"] != None:
self.updateZenossMessages(id)
for p in self.pairs:
id = p["id"]
zen = p["zenoss"]
if p["zenoss"] != None:
self.updateZenossStatus(id)
def updateZenossStatus(self, evid):
"""
update Zenoss event status if target is not current
"""
current = self.statusDict[evid]["current"]
target = self.statusDict[evid]["target"]
if current != target:
if self.options.verbose == True:
print "CHANGING STATUS %s IN ZENOSS TO %s" % (evid, target)
self.zenoss.manageEventStatus([evid],
self.statusData[target]['zenaction'])
def updateZenossMessages(self, evid):
"""
update Zenoss event console with messages if they don't exist
"""
eventlog = self.zenoss.getEventDetails(
evid)["result"]["event"][0]["log"]
logs = []
for e in eventlog:
logs.append(e[-1])
for msg in self.messenger.messages[evid]:
if msg not in logs:
self.zenoss.addEventMessage(evid, msg)
def getOptions(self):
"""
Command line runtime arguments
"""
usage = "usage: %prog [options] arg"
parser = OptionParser(usage)
# options for Zenoss
parser.add_option("-z",
"--zenhost",
dest="zenhost",
help="Zenoss Server hostname",
default=ZENOSS_HOST)
parser.add_option("-u",
"--zenuser",
dest="zenuser",
help="Zenoss admin username",
default=ZENOSS_USERNAME)
parser.add_option("-p",
"--zenpass",
dest="zenpass",
help="Zenoss admin password",
default=ZENOSS_PASSWORD)
parser.add_option("-e",
"--evid",
dest="evid",
help="Zenoss Event ID",
default=EVID)
parser.add_option("-V",
"--zenver",
dest="zenver",
help="True if Zenoss version >= 4",
action="store_false")
# options for Pagerduty
parser.add_option("-H",
"--pdhost",
dest="pdhost",
help="Pagerduty hostname",
default=PAGERDUTY_HOST)
parser.add_option("-T",
"--pdtoken",
dest="pdtoken",
help="Pagerduty token",
default=PAGERDUTY_TOKEN)
parser.add_option(
"-U",
"--pduser",
dest="pduser",
help="Pagerduty User Key (for tracking auto updates)",
default=PAGERDUTY_USER)
parser.add_option("-S",
"--servicekey",
dest="servicekey",
help="Pagerduty Service Key",
default=PAGERDUTY_SERVICEKEY)
# action to be performed
parser.add_option("-a",
"--action",
dest="action",
help="one of [create|update]",
default="update")
parser.add_option("-v",
"--verbose",
dest="verbose",
help="Show additional output",
action="store_true")
# options for zenoss interaction
(self.options, self.args) = parser.parse_args()
def run(self):
"""
control script execution
"""
if self.options.action == 'create':
self.messenger.newId(self.options.evid)
self.createPagerDutyIncident()
self.updateZenossMessages(self.options.evid)
self.updateZenossStatus(self.options.evid)
if self.options.action == 'update':
self.correlateByPagerDuty()
self.synchronize()
self.updatePagerDuty()
self.updateZenoss()
def _readMoreXML(self, xmlNode):
"""
Initializes data object based on XML input
@ In, xmlNode, xml.etree.ElementTree.Element or InputData.ParameterInput specification, input information
@ Out, None
"""
if isinstance(xmlNode, InputData.ParameterInput):
inp = xmlNode
else:
inp = DataObject.getInputSpecification()()
inp.parseNode(xmlNode)
# get hierarchical strategy
self.hierarchical = inp.parameterValues.get("hierarchical", False)
pivotParam = None # single pivot parameter given in the input
for child in inp.subparts:
# TODO check for repeats, "notAllowdInputs", names in both input and output space
if child.getName() == 'Input':
self._inputs.extend(
list(x.strip() for x in child.value.split(',')
if x.strip() != ''))
elif child.getName() == 'Output':
self._outputs.extend(
list(x.strip() for x in child.value.split(',')
if x.strip() != ''))
elif child.getName() == 'Index':
depends = list(d.strip() for d in child.value.split(','))
var = child.parameterValues['var']
self._pivotParams[var] = depends
# options node
elif child.getName() == 'options':
duplicateInp = False # if True, then multiple specification options were used for input
duplicateOut = False # if True, then multiple specification options were used for output
for cchild in child.subparts:
# pivot
if cchild.getName() == 'pivotParameter':
# TODO not applicable to ND, only to HistSet, but read it here
# TODO add checks somewhere if both "index" and "pivotParameter" are provided
self._tempPivotParam = cchild.value.strip()
# input pickers
elif cchild.getName() in ['inputRow', 'inputPivotValue']:
if self._selectInput is not None:
duplicateInp = True
self.setSelectiveInput(cchild.getName(), cchild.value)
# output pickers
elif cchild.getName() in [
'outputRow', 'outputPivotValue', 'operator'
]:
if self._selectOutput is not None:
duplicateOut = True
self._selectOutput = (cchild.getName(), cchild.value)
# TODO check this in the input checker instead of here?
if duplicateInp:
self.raiseAWarning(
'Multiple options were given to specify the input row to read! Using last entry:',
self._selectInput)
if duplicateOut:
self.raiseAWarning(
'Multiple options were given to specify the output row to read! Using last entry:',
self._selectOutput)
# end options node
# end input reading
# clear keywords InputPlaceHolder but NOT the OutputPlaceHolder, for legacy reasons
while 'InputPlaceHolder' in self._inputs:
self._inputs.remove('InputPlaceHolder')
#while 'OutputPlaceHolder' in self._outputs:
# self._outputs.remove('OutputPlaceHolder')
# set default pivot parameters, if needed
self._setDefaultPivotParams()
# remove index variables from input/output spaces, but silently, since we'll still have them available later
for index in self._pivotParams.keys():
try:
self._outputs.remove(index)
except ValueError:
pass #not requested as output anyway
try:
self._inputs.remove(index)
except ValueError:
pass #not requested as input anyway
self._orderedVars = self._inputs + self._outputs
# check if protected vars have been violated
if set(self.protectedTags).issubset(set(self._orderedVars)):
self.raiseAnError(
IOError,
'Input, Output and Index variables can not be part of RAVEN protected tags: '
+ ','.join(self.protectedTags))
# create dict var to index
# FIXME: this dict will not work in case of variables depending on multiple indexes. When this need comes, we will change this check(alfoa)
if self.indexes:
for ind in self.indexes:
self._fromVarToIndex.update(
dict.fromkeys(self._pivotParams[ind], ind))
if self.messageHandler is None:
self.messageHandler = MessageHandler()
range(int(start * 1. / step), int(end * 1. / step)))
if __name__ == "__main__":
param_file = sys.argv[ARG_PARAM_FILE]
eval_mode = sys.argv[ARG_EVAL_MODE]
min_twist = float(sys.argv[ARG_MIN_TWIST])
iter_twist = float(sys.argv[ARG_ITER_TWIST])
max_twist = float(sys.argv[ARG_MAX_TWIST])
min_grav = float(sys.argv[ARG_MIN_GRAV])
iter_grav = float(sys.argv[ARG_ITER_GRAV])
max_grav = float(sys.argv[ARG_MAX_GRAV])
print param_file, min_twist, iter_twist, max_twist, min_grav, iter_grav, max_grav
messenger = MessageHandler(HOST, PORT)
i = 0
for twist in frange(min_twist, max_twist + 20, 20):
BINARY_CMD = ("%s %s %s %s %s %s %s %s %s;" %
(BINARY, param_file, eval_mode, twist, iter_twist,
min(twist + 20 - iter_twist / 2,
max_twist), min_grav, iter_grav, max_grav))
job = "%s %s %s" % (COMPILE_CMD, BINARY_CMD, SCP_CMD)
print job
data = "USER:JOB:" + job
messenger.sendMessage(data)
time.sleep(0.1)
i += 1
messenger.close()
print i
port = 7777
try:
addr = str(sys.argv[sys.argv.index("-a") + 1])
port = int(sys.argv[sys.argv.index("-p") + 1])
except:
pass
#Creating chat and adding 6 users with given name
# new_chat = Chat("new")
# new_chat.add_user("Pavel")
# new_chat.add_user("Sveta")
# new_chat.add_user("Alex")
# new_chat.add_user("Girl")
# new_chat.add_user("Pavel_2")
# new_chat.add_user("Me")
req_handler = MessageHandler()
# req_handler.add_chat(new_chat)
req_handler.add_user("Pavel", "Password")
req_handler.add_user("Sveta", "Password")
req_handler.add_user("Alex", "Password")
req_handler.add_user("Girl", "Password")
req_handler.add_user("Pavel_2", "Password")
req_handler.add_user("Me", "Password")
server = Server((addr, port), req_handler)
server.run()
if not targetAlive:
print "User [" + targetID + "] not online"
return
message = raw_input("Enter the message:")
if ClientNetworkHelper.sendMsg(message, userid, targetID, destIPAddr):
print "Message send successfully"
def getUserLoginID():
#print
global userid
userid = raw_input("Enter your user id : ")
return userid
def addMessage(msgItem):
global msgList
msgList.append(msgItem)
if __name__ == "__main__":
showMainScreen()
userid = getUserLoginID()
print "\n\n Welcome " + userid + " !"
myIPAddr = ClientNetworkHelper.register(userid, Constants.trackerList)
if myIPAddr is None:
print "Failed to connect to any tracker / All trackers are full"
sys.exit(0)
mh = MessageHandler()
thread.start_new_thread(ClientNetworkHelper.recvThread, (mh, userid))
showMainOptions()
import SocketServer, commands, socket, time
from subprocess import Popen, PIPE, STDOUT
from logger import EC2Logger
from MessageHandler import MessageHandler
HOST = "128.32.37.215"
PORT = 10000
INSTANCE_ID = commands.getoutput("wget -q -O - http://169.254.169.254/latest/meta-data/instance-id")
PUBLIC_HOSTNAME = commands.getoutput("wget -q -O - http://169.254.169.254/latest/meta-data/public-hostname")
AGGREGATOR_CMD = "python /home/ubuntu/code/trunk/surgical/ec2/threaded_aggregator.py "
WORKER_CMD = "python /home/ubuntu/code/trunk/surgical/ec2/worker.py "
logger = EC2Logger("CFG-" + INSTANCE_ID)
messenger = MessageHandler(HOST, PORT)
data = INSTANCE_ID + "," + PUBLIC_HOSTNAME
logger.info(data+":READY")
msg = messenger.exchangeMessage(data+":READY")
messenger.close()
msg = msg.split(":")
if msg[0] == "AGGREGATOR":
logger.info("Starting Aggregator Service")
process = Popen("%s %s" % (AGGREGATOR_CMD, msg[1]), shell=True)
elif msg[0] == "WORKER":
logger.info("Starting Worker Service")
process = Popen("%s %s %s" % (WORKER_CMD, msg[1], msg[2]), shell=True)
while process.poll() is None:
time.sleep(15)
