def _parseActors(self, xml, vinstance):
logger().debug("VermontConfigurator._parseActors: called")
actors = xml.xpath("/ipfixConfig/actors/actor")
for a in actors:
idnumber = a.xpath("string(@id)")
action = a.xpath("string(action)")
if action == "disabled":
continue
code = a.xpath("string(code)")
trigger = a.xpath("string(trigger)")
delay = a.xpath("string(delay)")
target = a.xpath("string(target)")
foundit = False
for s in self._sensors:
if s.id == idnumber:
foundit = True
logger().debug(
"VermontConfigurator._parseActors: found new actor with id=%s"
% idnumber)
s.actors.append(
VermontActor(idnumber, action, code, trigger, delay,
target, vinstance))
if not foundit:
raise RuntimeError(
"failed to find sensor for actor with id=%s" % idnumber)
def _parseSensors(self, xml, vinstance):
logger().debug("VermontConfigurator._parseSensors: called")
sensors = xml.xpath("/ipfixConfig/sensors/sensor")
for s in sensors:
idnumber = s.xpath("string(@id)")
xpath = s.xpath("string(source)")
if xpath == "":
raise RuntimeError(
"failed to parse source for sensor with id=%s" % idnumber)
threshold = s.xpath("string(threshold)")
activation = s.xpath("string(activation)")
if (activation != "positive") and (activation != "negative"):
if activation == "disabled":
continue
raise RuntimeError("failed to parse element activation")
for cs in self._sensors:
if cs.id == idnumber:
raise RuntimeError(
"two sensor ids occured in configuration (id=%s)!" %
idnumber)
logger().debug(
"VermontConfigurator._parseSensors: found new sensor with id=%s"
% idnumber)
self._sensors.append(
VermontSensor(vinstance, idnumber, xpath, threshold,
activation))
def _workerThread(self):
while True:
logger().info("VermontController._workerThread: collecting monitoring data now")
self.rInterface.retrieveStatus()
if self.rInterface.running:
# try to read in statistics data several times ...
xml = None
trycount = 0
while xml is None:
#if trycount>=100:
#raise RuntimeError("Failed to read sensor data!")
try:
logger().debug("trying to read sensor data ...")
self.rInterface.retrieveSensorData()
xml = NonvalidatingReader.parseString(self.rInterface.sensorDataText)
except:
logger().error(traceback.format_exc())
logger().info("failed to read sensor data xml, trying again ...")
time.sleep(1)
trycount += 1
self.vMonitor.collect_data(xml)
else:
logger().info("VermontController._workerThread: skipping stat recording, as instance is not running")
time.sleep(self.moninterval)
def _changeModuleState(self, pausemodule):
moduleid = self.instance.dynCfgXml.xpath("number(%s/@id)" %
self.target)
if pausemodule:
refnext = self.instance.dynCfgXml.xpath(
"/ipfixConfig/*/next[number()=%d]" % moduleid)
for n in refnext:
logger().debug(
"VermontActor._performModifyValue: disabling node %s" %
n.xmlBase)
newnode = self.instance.dynCfgXml.createElementNS(
None, "next_disabled")
for c in n.childNodes:
newnode.appendChild(c)
p = n.parentNode
p.removeChild(n)
p.appendChild(newnode)
else:
refnext = self.instance.dynCfgXml.xpath(
"/ipfixConfig/*/next_disabled[number()=%d]" % moduleid)
for n in refnext:
logger().debug(
"VermontActor._performModifyValue: enabling node %s" %
n.xmlBase)
newnode = self.instance.dynCfgXml.createElementNS(None, "next")
for c in n.childNodes:
newnode.appendChild(c)
p = n.parentNode
p.removeChild(n)
p.appendChild(newnode)
self.instance.dynCfgModified = True
def reload(self):
logger().debug("LocalVermontInstance.reload()")
self.retrieveStatus()
if not self.running:
return
os.kill(self.pid, 10) # send SIGUSR1 to vermont
def _readConfig(self, configfile):
cp = ConfigParser.ConfigParser()
cp.read(configfile)
self.dir = cp.get("Global", "VermontDir")
self.cfgfile = cp.get("Global", "ConfigFile")
self.controllerlogfile = cp.get("Global", "ControllerLogFile")
#self.controllerlogfile = cp.get("Global", "/var/log/monkit/vcontroller.log")
self.logfile = cp.get("Global", "VermontLogFile")
if self.logfile.startswith("/"):
__cwd = os.getcwd() + self.dir
self.logfile = os.path.relpath(self.logfile, __cwd)
self.moninterval = int(cp.get("Stats", "Interval"))
self.allowedIp = cp.get("Global", "AllowedManagerIP")
self.bindAddress = cp.get("Global", "BindAddress")
self.listenPort = int(cp.get("Global", "ListenPort"))
logger().info("Using interval %s" % self.moninterval)
i = 1
self.xpaths = []
self.names = []
try:
while True:
self.xpaths.append(cp.get("Stats", "XPath_%d" % i))
self.names.append(cp.get("Stats", "Name_%d" % i))
i += 1
except: #IGNORE:W0704
pass
def stop(self):
logger().debug("LocalVermontInstance.stop()")
self.retrieveStatus()
if not self.running:
return True
try:
os.kill(self.pid, 2)
for _ in range(1, 15):
time.sleep(1)
pid, _ = os.waitpid(self.pid, os.WNOHANG)
if self.pid == pid:
self.pid = 0
return True
os.kill(self.pid, 9)
for _ in range(1, 10):
time.sleep(1)
pid, _ = os.waitpid(self.pid, os.WNOHANG)
if self.pid == pid:
self.pid = 0
return True
except:
logger().error(traceback.format_exc())
return False
def reload(self):
logger().debug("LocalVermontInstance.reload()")
self.retrieveStatus()
if not self.running:
return
os.kill(self.pid, 10) # send SIGUSR1 to vermont
def stop(self):
logger().debug("LocalVermontInstance.stop()")
self.retrieveStatus()
if not self.running:
return True
try:
os.kill(self.pid, 2)
for _ in range(1,15):
time.sleep(1)
pid, _ = os.waitpid(self.pid, os.WNOHANG)
if self.pid==pid:
self.pid = 0
return True
os.kill(self.pid, 9)
for _ in range(1,10):
time.sleep(1)
pid, _ = os.waitpid(self.pid, os.WNOHANG)
if self.pid==pid:
self.pid = 0
return True
except:
logger().error(traceback.format_exc())
return False
def setup(self):
logger().debug("VermontInstanceManager.setup()")
self._startConfigThread()
# RPC server
self._server = VMRPCServer((self._bindAddress, self._listenPort), allow_none=True, logRequests=False)
self._server.allow_reuse_address = True
self._server.allowedIp = self._allowedIP # IGNORE:W0201
self._server.register_instance(self)
def collect_data(self, xml):
if not os.access("rrd", os.R_OK|os.W_OK):
os.mkdir("rrd")
rrdfn = "rrd/db_%d_%d.rrd"
for i in range(0, len(self.xpaths)):
data = xml.xpath(self.xpaths[i])
logger().info("inserting value %s for element '%s' (%s)" % (data, self.names[i], self.xpaths[i]))
for j in range(0, len(self.rrdintervals)):
if not os.access(rrdfn % (i, j), os.R_OK|os.W_OK):
os.system("rrdtool create %s -s 5 DS:s:GAUGE:%d:U:U RRA:AVERAGE:0.5:%d:10000 RRA:MIN:0.5:%d:10000 RRA:MAX:0.5:%d:10000" % (rrdfn % (i, j), self.rrdintervals[j]*5*2, self.rrdintervals[j], self.rrdintervals[j], self.rrdintervals[j]))
os.system("rrdtool update %s N:%f " % (rrdfn % (i, j), data))
def retrieveConfig(self):
"""
retrieves original configuration from vermont instance and replaces current configurations,
both original and dynamic
"""
logger().debug("VermontInstance.retrieveConfig()")
try:
self.setConfig(self._retrieveConfig())
self.__cfgModified = False
except:
logger().error(traceback.format_exc())
def retrieveConfig(self):
"""
retrieves original configuration from vermont instance and replaces current configurations,
both original and dynamic
"""
logger().debug("VermontInstance.retrieveConfig()")
try:
self.setConfig(self._retrieveConfig())
self.__cfgModified = False
except:
logger().error(traceback.format_exc())
def setConfig(self, text):
"""
sets new original configuration xml in this instances, recreates dynamic configuration
"""
logger().debug("VermontInstance.setConfig()")
self.__cfgModified = True
self.__cfgText = text
if self.__parseXml:
self.__cfgXml = NonvalidatingReader.parseString(text)
self.__dynCfgModified = True
self.__dynCfgText = self.cfgText
if self.__parseXml:
self.__dynCfgXml = NonvalidatingReader.parseString(self.__cfgText)
def setConfig(self, text):
"""
sets new original configuration xml in this instances, recreates dynamic configuration
"""
logger().debug("VermontInstance.setConfig()")
self.__cfgModified = True
self.__cfgText = text
if self.__parseXml:
self.__cfgXml = NonvalidatingReader.parseString(text)
self.__dynCfgModified = True
self.__dynCfgText = self.cfgText
if self.__parseXml:
self.__dynCfgXml = NonvalidatingReader.parseString(self.__cfgText)
def syncConfig(self):
"""
synchronizes both original and dynamic configuration to vermont instance
"""
logger().debug("VermontInstance.syncConfig()")
if self.__cfgModified:
self._transmitConfig()
self.__cfgModified = False
if self.dynCfgModified:
if self.__parseXml:
textio = StringIO()
Print(self.dynCfgXml, stream=textio)
self.__dynCfgText = textio.getvalue()
self._transmitDynConfig()
self.__dynCfgModified = False
def syncConfig(self):
"""
synchronizes both original and dynamic configuration to vermont instance
"""
logger().debug("VermontInstance.syncConfig()")
if self.__cfgModified:
self._transmitConfig()
self.__cfgModified = False
if self.dynCfgModified:
if self.__parseXml:
textio = StringIO()
Print(self.dynCfgXml, stream=textio)
self.__dynCfgText = textio.getvalue()
self._transmitDynConfig()
self.__dynCfgModified = False
def collect_data(self, xml):
if not os.access("rrd", os.R_OK | os.W_OK):
os.mkdir("rrd")
rrdfn = "rrd/db_%d_%d.rrd"
for i in range(0, len(self.xpaths)):
data = xml.xpath(self.xpaths[i])
logger().info("inserting value %s for element '%s' (%s)" %
(data, self.names[i], self.xpaths[i]))
for j in range(0, len(self.rrdintervals)):
if not os.access(rrdfn % (i, j), os.R_OK | os.W_OK):
os.system(
"rrdtool create %s -s 5 DS:s:GAUGE:%d:U:U RRA:AVERAGE:0.5:%d:10000 RRA:MIN:0.5:%d:10000 RRA:MAX:0.5:%d:10000"
% (rrdfn % (i, j), self.rrdintervals[j] * 5 * 2,
self.rrdintervals[j], self.rrdintervals[j],
self.rrdintervals[j]))
os.system("rrdtool update %s N:%f " % (rrdfn % (i, j), data))
def retrieveStatus(self):
if self.pid != 0:
# cleanup zombie process, if found ...
try:
(pid, ) = os.waitpid(self.pid, os.WNOHANG)
if self.pid==pid:
self.pid = 0
except: #IGNORE:W0704
pass
if os.access("/proc/%d" % self.pid, os.F_OK):
self.running = True
else:
self.running = False
self.pid = 0
else:
self.running = False
logger().debug("LocalVermontInstance.retrieveStatus: instance running: %s, pid: %d" % (self.running, self.pid))
def _parseSensors(self, xml, vinstance):
logger().debug("VermontConfigurator._parseSensors: called")
sensors = xml.xpath("/ipfixConfig/sensors/sensor")
for s in sensors:
idnumber = s.xpath("string(@id)")
xpath = s.xpath("string(source)")
if xpath=="":
raise RuntimeError("failed to parse source for sensor with id=%s" % idnumber)
threshold = s.xpath("string(threshold)")
activation = s.xpath("string(activation)")
if (activation!="positive") and (activation!="negative"):
if activation=="disabled":
continue
raise RuntimeError("failed to parse element activation")
for cs in self._sensors:
if cs.id==idnumber:
raise RuntimeError("two sensor ids occured in configuration (id=%s)!" % idnumber)
logger().debug("VermontConfigurator._parseSensors: found new sensor with id=%s" % idnumber)
self._sensors.append(VermontSensor(vinstance, idnumber, xpath, threshold, activation))
def retrieveStatus(self):
if self.pid != 0:
# cleanup zombie process, if found ...
try:
(pid, ) = os.waitpid(self.pid, os.WNOHANG)
if self.pid == pid:
self.pid = 0
except: #IGNORE:W0704
pass
if os.access("/proc/%d" % self.pid, os.F_OK):
self.running = True
else:
self.running = False
self.pid = 0
else:
self.running = False
logger().debug(
"LocalVermontInstance.retrieveStatus: instance running: %s, pid: %d"
% (self.running, self.pid))
def __init__(self, configfile):
self.dir = None
self.cfgfile = None
self.logfile = None
self.moninterval = None
self.allowedIp = None
self.xpaths = []
self.names = []
self.server = None
self.vMonitor = None
self.rInterface = None
self.controllerlogfile = None
try:
VermontLogger.init("vcontroller", "", True)
self._readConfig(configfile)
VermontLogger.init("vcontroller", self.controllerlogfile, True)
except:
print "failed to read configuration file!"
logger().error(traceback.format_exc())
def _parseActors(self, xml, vinstance):
logger().debug("VermontConfigurator._parseActors: called")
actors = xml.xpath("/ipfixConfig/actors/actor")
for a in actors:
idnumber = a.xpath("string(@id)")
action = a.xpath("string(action)")
if action=="disabled":
continue
code = a.xpath("string(code)")
trigger = a.xpath("string(trigger)")
delay = a.xpath("string(delay)")
target = a.xpath("string(target)")
foundit = False
for s in self._sensors:
if s.id==idnumber:
foundit = True
logger().debug("VermontConfigurator._parseActors: found new actor with id=%s" % idnumber)
s.actors.append(VermontActor(idnumber, action, code, trigger, delay, target, vinstance))
if not foundit:
raise RuntimeError("failed to find sensor for actor with id=%s" % idnumber)
def _changeModuleState(self, pausemodule):
moduleid = self.instance.dynCfgXml.xpath("number(%s/@id)" % self.target)
if pausemodule:
refnext = self.instance.dynCfgXml.xpath("/ipfixConfig/*/next[number()=%d]" % moduleid)
for n in refnext:
logger().debug("VermontActor._performModifyValue: disabling node %s" % n.xmlBase)
newnode = self.instance.dynCfgXml.createElementNS(None, "next_disabled")
for c in n.childNodes: newnode.appendChild(c)
p = n.parentNode
p.removeChild(n)
p.appendChild(newnode)
else:
refnext = self.instance.dynCfgXml.xpath("/ipfixConfig/*/next_disabled[number()=%d]" % moduleid)
for n in refnext:
logger().debug("VermontActor._performModifyValue: enabling node %s" % n.xmlBase)
newnode = self.instance.dynCfgXml.createElementNS(None, "next")
for c in n.childNodes: newnode.appendChild(c)
p = n.parentNode
p.removeChild(n)
p.appendChild(newnode)
self.instance.dynCfgModified = True
def start(self):
logger().debug("LocalVermontInstance.start()")
self.retrieveStatus()
if self.running:
return
cmdline = ["vermont", "-f", self.dyncfgfile, "-d"]
logfile = "%s/%s" % (self.dir, self.logfile)
logger().info("Vermont args: %s" % cmdline)
logger().info("Truncating log file %s" % self.logfile)
f = os.open(self.logfile, os.O_CREAT | os.O_TRUNC)
os.close(f)
pid = os.fork()
if pid == 0:
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (maxfd == resource.RLIM_INFINITY):
maxfd = os.sysconf('SC_OPEN_MAX')
REDIRECT_TO = self.logfile
# Iterate through and close all file descriptors.
for fd in range(0, maxfd):
try:
os.close(fd)
except: #IGNORE:W0704
pass
os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
os.chdir(self.dir)
try:
os.execv("./vermont", cmdline)
except:
logger().error(traceback.format_exc())
sys.stderr.write(
"Problem starting Vermont, exiting forked process ...\n")
os._exit(1)
else:
logger().debug("PID of started vermont: %d" % pid)
self.pid = pid
def start(self):
logger().debug("LocalVermontInstance.start()")
self.retrieveStatus()
if self.running:
return
cmdline = [ "vermont", "-f", self.dyncfgfile, "-d" ]
logfile = "%s/%s" % (self.dir, self.logfile)
logger().info("Vermont args: %s" % cmdline)
logger().info("Truncating log file %s" % self.logfile)
f = os.open(self.logfile, os.O_CREAT|os.O_TRUNC)
os.close(f)
pid = os.fork()
if pid == 0:
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (maxfd == resource.RLIM_INFINITY):
maxfd = os.sysconf('SC_OPEN_MAX')
REDIRECT_TO = self.logfile
# Iterate through and close all file descriptors.
for fd in range(0, maxfd):
try:
os.close(fd)
except: #IGNORE:W0704
pass
os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
os.chdir(self.dir)
try:
os.execv("./vermont", cmdline)
except:
logger().error(traceback.format_exc())
sys.stderr.write("Problem starting Vermont, exiting forked process ...\n")
os._exit(1)
else:
logger().debug("PID of started vermont: %d" % pid)
self.pid = pid
def _performModifyValue(self):
nodes = self.instance.dynCfgXml.xpath(self.target)
if len(nodes) > 1:
logger().warning(
"WARNING: found more than one matching node for xpath string '%s'"
% self.target)
targetnode = None
for n in nodes[0].childNodes:
if isinstance(n, Ft.Xml.cDomlette.Text):
targetnode = n
break
if targetnode is None:
raise RuntimeError(
"failed to find target node for actor with id=%s" % self.id)
vrs = {'v': targetnode.nodeValue}
logger().info("old v: %s" % vrs['v'])
logger().info("code: %s" % self.code)
try:
exec self.code in vrs # IGNORE:W0122
except:
logger().error(
"failed to execute code in instance %s for actor id=%d" %
(self.instance, self.id))
logger().error(traceback.format_exc())
logger().info("new v: %s" % vrs['v'])
if vrs['v'] != targetnode.nodeValue:
targetnode.nodeValue = str(vrs['v'])
self.instance.dynCfgModified = True
def retrieveSensorData(self):
logger().debug("VermontInstance.retrieveSensorData()")
self.__sensorDataText = self._retrieveSensorData()
if self.__parseXml:
self.__sensorDataXml = NonvalidatingReader.parseString(self.__sensorDataText)
def _startConfigThread(self):
"starts internal reconfiguration thread"
logger().debug("VermontInstanceManager._startConfigThread()")
self._workerRunning = True
thread.start_new_thread(VermontInstanceManager.configThreadWorker, (self, ))
def _retrieveSensorData(self):
logger().debug("RemoteVermontInstance._retrieveSensorData()")
self._conn.retrieveSensorData()
return self._conn.getSensorDataText()
def setDynCfgText(self, value):
logger().debug("setting new dyncfgtext")
self.__dynCfgText = value
def setCfgText(self, value):
logger().debug("setting new cfgtext")
self.__cfgText = value
def start(self):
try:
LocalVermontInstance.start(self)
except:
logger().error(traceback.format_exc())
raise
def setCfgText(self, value):
logger().debug("setting new cfgtext")
self.__cfgText = value
def configThreadWorker(self):
"internal worker thread that regularly checks sensors, actors and parses configuration data"
logger().info("VermontInstanceManager.configThreadWorker: starting")
while self.__dynconfEnabled:
try:
for vi in self.vermontInstances:
try:
vi.retrieveStatus()
if vi.running:
logger().debug("getting sensor data for instance %s" % vi)
vi.retrieveSensorData()
except:
logger().error(traceback.format_exc())
logger().debug("checking sensors")
self._configurator.checkSensors()
logger().debug("saving modified configurations to vermont")
for vi in self.vermontInstances:
if vi.dynCfgModified:
try:
logger().debug("syncing dynamic config and reloading vermont instance %s" % vi)
vi.syncConfig()
vi.reload()
except:
logger().error("error occured while saving config for vermont instance %s" % vi)
logger().error(traceback.format_exc())
logger().debug("sleeping %d seconds" % self._checkInterval)
except:
logger().error("error occured in worker thread")
logger().error(traceback.format_exc())
time.sleep(self._checkInterval)
self._workerRunning = False
logger().info("VermontInstanceManager.configThreadWorker: stopping")
def _performModifyValue(self):
nodes = self.instance.dynCfgXml.xpath(self.target)
if len(nodes)>1:
logger().warning("WARNING: found more than one matching node for xpath string '%s'" % self.target)
targetnode = None
for n in nodes[0].childNodes:
if isinstance(n, Ft.Xml.cDomlette.Text):
targetnode = n
break
if targetnode is None:
raise RuntimeError("failed to find target node for actor with id=%s" % self.id)
vrs = { 'v' : targetnode.nodeValue}
logger().info("old v: %s" % vrs['v'])
logger().info("code: %s" % self.code)
try:
exec self.code in vrs # IGNORE:W0122
except:
logger().error("failed to execute code in instance %s for actor id=%d" % (self.instance, self.id))
logger().error(traceback.format_exc())
logger().info("new v: %s" % vrs['v'])
if vrs['v']!=targetnode.nodeValue:
targetnode.nodeValue = str(vrs['v'])
self.instance.dynCfgModified = True
def parseConfigs(self, vinstances):
"""parses given list of VermontInstances for sensors/actors and recreates internal structure
@param vinstances:
@ptype vinstances: array of VermontInstance
"""
logger().info("VermontConfigurator.parseConfigs: reparsing configuration for sensors and actors")
self._sensors = []
for v in vinstances:
if v.cfgXml is None:
try:
v.retrieveConfig()
except:
logger().error(traceback.format_exc())
if v.cfgXml is not None:
try:
self._parseSensors(v.cfgXml, v)
except:
logger().warning("failed to parse configuration from instance %s" % v)
logger().error(traceback.format_exc())
else:
logger().debug("failed to get configuration for vermont instance %s" % v)
for v in vinstances:
if v.cfgXml is not None:
try:
self._parseActors(v.cfgXml, v)
except:
logger().warning("failed to parse configuration from instance %s" % v)
logger().error(traceback.format_exc())
def retrieveSensorData(self):
logger().debug("VermontInstance.retrieveSensorData()")
self.__sensorDataText = self._retrieveSensorData()
if self.__parseXml:
self.__sensorDataXml = NonvalidatingReader.parseString(
self.__sensorDataText)
def setDynCfgText(self, value):
logger().debug("setting new dyncfgtext")
self.__dynCfgText = value
def start(self):
try:
LocalVermontInstance.start(self)
except:
logger().error(traceback.format_exc())
raise
def parseConfigs(self, vinstances):
"""parses given list of VermontInstances for sensors/actors and recreates internal structure
@param vinstances:
@ptype vinstances: array of VermontInstance
"""
logger().info(
"VermontConfigurator.parseConfigs: reparsing configuration for sensors and actors"
)
self._sensors = []
for v in vinstances:
if v.cfgXml is None:
try:
v.retrieveConfig()
except:
logger().error(traceback.format_exc())
if v.cfgXml is not None:
try:
self._parseSensors(v.cfgXml, v)
except:
logger().warning(
"failed to parse configuration from instance %s" % v)
logger().error(traceback.format_exc())
else:
logger().debug(
"failed to get configuration for vermont instance %s" % v)
for v in vinstances:
if v.cfgXml is not None:
try:
self._parseActors(v.cfgXml, v)
except:
logger().warning(
"failed to parse configuration from instance %s" % v)
logger().error(traceback.format_exc())
