def test(self, service):
def Test(driver):
data = ('Some Data', 17)
yield service.callRemote('echo', data)
self.tester.assertEqual(driver.next(), data)
self.success = True
drive(Test).addBoth(stop, connector=self.connector)
def test(self, service):
def Test(driver):
data = ('Some Data', 17)
yield service.callRemote('echo', data)
self.tester.assertEqual(driver.next(), data)
self.success = True
drive(Test).addBoth(stop, connector=self.connector)
def test(self, service):
def Test(driver):
evt = dict(device='localhost',
severity='5',
summary='This is a test message')
yield service.callRemote('sendEvents', [evt])
self.tester.assertEqual(driver.next(), 1)
self.success = True
drive(Test).addBoth(stop, connector=self.connector)
def test(self, service):
def Test(driver):
evt = dict(device='localhost',
severity='5',
summary='This is a test message')
yield service.callRemote('sendEvents', [evt])
self.tester.assertEqual(driver.next(), 1)
self.success = True
drive(Test).addBoth(stop, connector=self.connector)
def startDiscovery(self, data):
if self.options.walk:
d = drive(self.walkDiscovery)
elif self.options.device:
d = drive(self.createDevice)
elif self.options.range:
d = drive(self.discoverRanges)
else:
d = drive(self.collectNet)
d.addBoth(self.printResults)
def startDiscovery(self, data):
if self.options.walk:
d = drive(self.walkDiscovery)
elif self.options.device:
d = drive(self.createDevice)
elif self.options.range:
d = drive(self.discoverRanges)
else:
d = drive(self.collectNet)
d.addBoth(self.printResults)
def collect(self, device, log):
"""Return deferred with results of collection."""
log.info("%s: collecting client MAC addresses", device.id)
# Inspect MAC addresses format in zLocalMacAddresses
for mac in device.zLocalMacAddresses:
if not is_valid_macaddr802(mac):
log.warn("Invalid MAC Address '%s' found in %s", mac,
'zLocalMacAddresses')
iftable = copy.deepcopy(getattr(device, 'get_ifinfo_for_layer2', {}))
state = ClientMACsState(device=device, iftable=iftable, log=log)
results = []
for community in state.all_communities:
snmp_client = state.get_snmp_client(community=community)
try:
yield drive(snmp_client.doRun)
except Exception:
# Error will be logged at INFO by SnmpClient.
pass
else:
client_results = snmp_client.getResults()
for plugin, (_, tabledata) in client_results:
results.append(
{x.name: x.mapdata(y)
for x, y in tabledata.items()})
finally:
snmp_client.stop()
returnValue((state, results))
def fetchSome(self, timeoutMs=wbemTimeoutInfinite, chunkSize=10,
includeQualifiers=False):
assert self.pEnum
def inner(driver):
count = uint32_t()
objs = (POINTER(WbemClassObject)*chunkSize)()
ctx = library.IEnumWbemClassObject_SmartNext_send(
self.pEnum, None, timeoutMs, chunkSize
)
yield deferred(ctx); driver.next()
result = library.IEnumWbemClassObject_SmartNext_recv(
ctx, self.ctx, objs, byref(count)
)
WERR_CHECK(result, self._deviceId, "Retrieve result data.")
result = []
if includeQualifiers:
for i in range(count.value):
result.append(wbemInstanceWithQualifiersToPython(objs[i]))
talloc_free(objs[i])
else:
for i in range(count.value):
result.append(wbemInstanceToPython(objs[i]))
talloc_free(objs[i])
driver.finish(result)
return drive(inner)
def processClientFinished(result):
"""
Called after the client collection finishes
@param result: object (unused)
@type result: object
"""
self.counters['modeledDevicesCount'] += 1
self._modeledDevicesMetric.mark()
# result is now the result of remote_applyDataMaps (from processClient)
if result and isinstance(result, (basestring, Failure)):
self.log.error("Client %s finished with message: %s" %
(device.id, result))
self._failuresMetric.increment()
else:
self.log.debug("Client %s finished" % device.id)
try:
self.clients.remove(collectorClient)
self.finished.append(collectorClient)
except ValueError:
self.log.debug("Client %s not found in in the list"
" of active clients",
device.id)
self.log.info("Finished processing client within collector loop #{0:03d}".format(self.collectorLoopIteration))
d = drive(self.fillCollectionSlots)
d.addErrback(self.fillError)
def connect(self, eventContext, deviceId, hostname, creds, namespace="root\\cimv2"):
self._deviceId = deviceId
library.com_init_ctx(byref(self.ctx), eventContext)
cred = library.cli_credentials_init(self.ctx)
library.cli_credentials_set_conf(cred)
library.cli_credentials_parse_string(cred, creds, CRED_SPECIFIED)
library.dcom_client_init(self.ctx, cred)
def inner(driver):
flags = uint32_t()
flags.value = 0
ctx = library.WBEM_ConnectServer_send(
self.ctx, # com_ctx
None, # parent_ctx
hostname, # server
namespace, # namespace
None, # username
None, # password
None, # locale
flags.value, # flags
None, # authority
None) # wbem_ctx
yield deferred(ctx); driver.next()
result = library.WBEM_ConnectServer_recv(ctx, None, byref(self.pWS))
WERR_CHECK(result, self._deviceId, "Connect")
driver.finish(None)
return drive(inner)
def collect(self, device, log):
"""Return deferred with results of collection."""
log.info("%s: collecting client MAC addresses", device.id)
self.log = log
state = ClientMACsState(device=device,
iftable=getattr(device,
'get_ifinfo_for_layer2', {}),
macs_indexed=getattr(device, 'macs_indexed',
False))
results = []
for community in state.snmp_communities():
snmp_client = state.snmp_client(community=community)
try:
yield drive(snmp_client.doRun)
except Exception:
# Error will be logged at INFO by SnmpClient.
pass
else:
client_results = snmp_client.getResults()
for plugin, (_, tabledata) in client_results:
results.append(
{x.name: x.mapdata(y)
for x, y in tabledata.items()})
finally:
snmp_client.stop()
returnValue((state, results))
def fetch(self):
"""
Main data collection routine, loops though data sources, updating each one
after executing datasource commands
"""
def inner(driver):
result = {}
log.debug("Executing commands for %s (%s datasources)", self._devId,len(self._taskConfig.datasources.items()))
for compId, dsDict in self._taskConfig.datasources.items():
self._compId = compId
result[compId] = {}
log.debug("Collecting data for component: %s on device: %s",compId,self._devId)
for dsId, compDict in dsDict.items():
self._dsId = dsId
result[compId][dsId] = {}
command = compDict['command']
sourcetype = compDict['sourcetype']
log.debug("Executing command: %s for type: %s",command,sourcetype)
output = self.siebelClient.getCommandOutput(command)
if len(output) == 1 and 'CP_DISP_RUN_STATE' in command:
result[compId][dsId]['runState'] = self.getRunState(output[0])
elif 'list tasks' in command:
numTasks,goodTasks,badTasks = self.getTasks(output)
result[compId][dsId]['numTasks'] = numTasks
result[compId][dsId]['goodTasks'] = goodTasks
result[compId][dsId]['badTasks'] = badTasks
else:
dataItems = self.siebelClient.statDict(output,'STAT_ALIAS','CURR_VAL')
for dpId,dpValue in dataItems.items():
result[compId][dsId][dpId] = dpValue
yield defer.succeed(result)
return drive(inner)
def collect(self, device, log):
"""Return deferred with results of collection."""
log.info("%s: collecting client MAC addresses", device.id)
# Inspect MAC addresses format in zLocalMacAddresses
for mac in device.zLocalMacAddresses:
if not is_valid_macaddr802(mac):
log.warn("Invalid MAC Address '%s' found in %s", mac, 'zLocalMacAddresses')
iftable = copy.deepcopy(getattr(device, 'get_ifinfo_for_layer2', {}))
state = ClientMACsState(
device=device,
iftable=iftable,
log=log)
results = []
for community in state.all_communities:
snmp_client = state.get_snmp_client(community=community)
try:
yield drive(snmp_client.doRun)
except Exception:
# Error will be logged at INFO by SnmpClient.
pass
else:
client_results = snmp_client.getResults()
for plugin, (_, tabledata) in client_results:
results.append(
{x.name: x.mapdata(y) for x, y in tabledata.items()})
finally:
snmp_client.stop()
returnValue((state, results))
def startEmailCycle( self, ignored = None ):
deferred = drive( self.emailCycle )
deferred.addErrback( self.logError )
if not self.options.cycle:
deferred.addBoth( lambda unused: self.stop() )
def processClientFinished(result):
"""
Called after the client collection finishes
@param result: object (unused)
@type result: object
"""
self.counters['modeledDevicesCount'] += 1
self._modeledDevicesMetric.mark()
# result is now the result of remote_applyDataMaps (from processClient)
if result and isinstance(result, (basestring, Failure)):
self.log.error("Client %s finished with message: %s" %
(device.id, result))
self._failuresMetric.increment()
else:
self.log.debug("Client %s finished" % device.id)
try:
self.clients.remove(collectorClient)
self.finished.append(collectorClient)
except ValueError:
self.log.debug(
"Client %s not found in in the list"
" of active clients", device.id)
self.log.info(
"Finished processing client within collector loop #{0:03d}".
format(self.collectorLoopIteration))
d = drive(self.fillCollectionSlots)
d.addErrback(self.fillError)
def getMessage(config, pollSeconds, lines=0):
"Poll a pop account for the message that goes with this config"
if config.msgid is None:
return defer.fail(
ValueError("No outstanding message for %s:%s" %
(config.device, config.name)))
start = time.time()
end = start + config.timeout
def poll(driver):
while 1:
remaining = end - time.time()
if remaining < 0:
raise TimeoutError
yield fetchOnce(config, lines)
try:
if driver.next() is not None:
yield defer.succeed(driver.next())
config.msgid = None
break
except ConnectionLost:
pass
remaining = end - time.time()
if remaining < 0:
raise TimeoutError
yield timeout(min(remaining, pollSeconds))
driver.next()
return drive(poll)
def fetchConfig(self):
"""
Get configuration values from ZenHub
"""
def inner(driver):
self.log.debug("fetchConfig(): Fetching config from zenhub")
yield self.model().callRemote('getDefaultRRDCreateCommand')
createCommand = driver.next()
yield self.model().callRemote('propertyItems')
self.setPropertyItems(driver.next())
self.rrd = RRDUtil(createCommand, DEFAULT_HEARTBEAT_TIME)
yield self.model().callRemote('getThresholdClasses')
self.remote_updateThresholdClasses(driver.next())
yield self.model().callRemote('getCollectorThresholds')
self.rrdStats.config(self.options.monitor,
self.name,
driver.next(),
createCommand)
devices=self.getDevices()
instances=self.getInstances()
yield self.model().callRemote('getDeviceConfigs', instances, devices )
configs = driver.next()
self.log.debug('Fetched %i configs' % len( configs ) )
if len(configs) == 0:
self.log.info("fetchConfig(): No configs returned from zenhub")
else:
for instance in configs.keys():
deviceConfigs=configs[instance]
self.updateConfig(instance, deviceConfigs)
self.log.debug("fetchConfig(): Done fetching config from zenhub")
return drive(inner)
def fetchSome(self,
timeoutMs=wbemTimeoutInfinite,
chunkSize=10,
includeQualifiers=False):
assert self.pEnum
def inner(driver):
count = uint32_t()
objs = (POINTER(WbemClassObject) * chunkSize)()
ctx = library.IEnumWbemClassObject_SmartNext_send(
self.pEnum, None, timeoutMs, chunkSize)
yield deferred(ctx)
driver.next()
result = library.IEnumWbemClassObject_SmartNext_recv(
ctx, self.ctx, objs, byref(count))
WERR_CHECK(result, self._deviceId, "Retrieve result data.")
result = []
if includeQualifiers:
for i in range(count.value):
result.append(wbemInstanceWithQualifiersToPython(objs[i]))
talloc_free(objs[i])
else:
for i in range(count.value):
result.append(wbemInstanceToPython(objs[i]))
talloc_free(objs[i])
driver.finish(result)
return drive(inner)
def mainLoop(self, driver):
"""
Main collection loop, a Python iterable
@param driver: driver object
@type driver: driver object
@return: Twisted deferred object
@rtype: Twisted deferred object
"""
if self.options.cycle:
driveLater(self.cycleTime(), self.mainLoop)
if self.clients:
self.log.error("Modeling cycle taking too long")
return
self.start = time.time()
self.log.debug("Starting collector loop...")
yield self.getDeviceList()
self.devicegen = iter(driver.next())
d = drive(self.fillCollectionSlots)
d.addErrback(self.fillError)
yield d
driver.next()
self.log.debug("Collection slots filled")
def connect(self, eventContext, deviceId, hostname, creds, namespace="root\\cimv2"):
self._deviceId = deviceId
library.com_init_ctx.restype = WERROR
library.com_init_ctx(byref(self.ctx), eventContext)
cred = library.cli_credentials_init(self.ctx)
library.cli_credentials_set_conf(cred)
library.cli_credentials_parse_string(cred, creds, CRED_SPECIFIED)
library.dcom_client_init(self.ctx, cred)
def inner(driver):
flags = uint32_t()
flags.value = 0
ctx = library.WBEM_ConnectServer_send(
self.ctx, # com_ctx
None, # parent_ctx
hostname, # server
namespace, # namespace
None, # username
None, # password
None, # locale
flags.value, # flags
None, # authority
None,
) # wbem_ctx
yield deferred(ctx)
driver.next()
result = library.WBEM_ConnectServer_recv(ctx, None, byref(self.pWS))
WERR_CHECK(result, self._deviceId, "Connect")
driver.finish(None)
return drive(inner)
def run(self):
def inner(driver):
tasks = {}
for plugin in self.plugins:
pn = plugin.name()
for tn, t in (plugin.prepareQueries(self.device)
or {}).iteritems():
tasks[(pn, tn)] = t
yield defer.maybeDeferred(self.query, tasks, sync=False)
driver.next()
d = drive(inner)
def finish(results):
if isinstance(results, Failure):
for pl in self.plugins:
self.results.append((pl, results))
else:
for pl in self.plugins:
self.results.append((pl, results.pop(pl.name(), {})))
if self.datacollector:
self.datacollector.clientFinished(self)
else:
from twisted.internet import reactor
reactor.stop()
d.addBoth(finish)
return d
def mainLoop(self, driver):
"""
Main collection loop, a Python iterable
@param driver: driver object
@type driver: driver object
@return: Twisted deferred object
@rtype: Twisted deferred object
"""
if self.options.cycle:
driveLater(self.cycleTime(), self.mainLoop)
if self.clients:
self.log.error("Modeling cycle taking too long")
return
self.start = time.time()
self.log.debug("Starting collector loop...")
yield self.getDeviceList()
self.devicegen = iter(driver.next())
d = drive(self.fillCollectionSlots)
d.addErrback(self.fillError)
yield d
driver.next()
self.log.debug("Collection slots filled")
def fetchConfig(self):
"""
Get configuration values from ZenHub
"""
def inner(driver):
self.log.debug("fetchConfig(): Fetching config from zenhub")
yield self.model().callRemote('getDefaultRRDCreateCommand')
createCommand = driver.next()
yield self.model().callRemote('propertyItems')
self.setPropertyItems(driver.next())
self.rrd = RRDUtil(createCommand, DEFAULT_HEARTBEAT_TIME)
yield self.model().callRemote('getThresholdClasses')
self.remote_updateThresholdClasses(driver.next())
yield self.model().callRemote('getCollectorThresholds')
self.rrdStats.config(self.options.monitor, self.name,
driver.next(), createCommand)
devices = self.getDevices()
instances = self.getInstances()
yield self.model().callRemote('getDeviceConfigs', instances,
devices)
configs = driver.next()
self.log.debug('Fetched %i configs' % len(configs))
if len(configs) == 0:
self.log.info("fetchConfig(): No configs returned from zenhub")
else:
for instance in configs.keys():
deviceConfigs = configs[instance]
self.updateConfig(instance, deviceConfigs)
self.log.debug("fetchConfig(): Done fetching config from zenhub")
return drive(inner)
def collect(self, config):
"""
Iterates over device's Ip Interfaces and gathers Layer 2 information
with SNMP
"""
results = self.new_data()
ds0 = config.datasources[0]
ds0.id = config.id
self.iftable = ds0.get_ifinfo_for_layer2
self.macs_indexed = ds0.macs_indexed
self.jobs = []
self.community = ds0.zSnmpCommunity
for vlan in self.get_vlans(): # ["1", "951"]:
sc = self.get_snmp_client(vlan, config, ds0)
yield drive(sc.doRun)
self._prep_iftable(self.get_snmp_data(sc))
sc.stop()
results['maps'] = self.get_maps()
defer.returnValue(results)
def periodic(self, unused=None):
"""
Main loop that drives all other processing.
"""
reactor.callLater(self.processCycleInterval, self.periodic)
if self.scanning:
running, unstarted, finished = self.scanning.status()
runningDevices = [ d.name for d in self.devices().values() \
if d.proxy is not None]
if runningDevices or unstarted > 0:
log.warning("Process scan not finishing: "
"%d running, %d waiting, %d finished" % (
running, unstarted, finished))
log.warning("Problem devices: %r", runningDevices)
return
start = time.time()
def doPeriodic(driver):
"""
Generator function to create deferred jobs.
"""
yield self.getDevicePingIssues()
self.downDevices = Set([d[0] for d in driver.next()])
self.scanning = NJobs(self.parallelJobs,
self.oneDevice,
self.devices().values())
yield self.scanning.start()
driver.next()
def checkResults(results):
"""
Process the results from all deferred objects.
"""
for result in results:
if isinstance(result , Exception):
log.error("Error scanning device: %s", result)
break
self.cycleTime = time.time() - start
self.heartbeat()
drive(doPeriodic).addCallback(checkResults)
def connected(self):
def configTask(driver):
self.log.debug("configTask(): fetching config")
yield self.fetchConfig()
driver.next()
driveLater(self.configCycleInterval*60, configTask)
d = drive(configTask)
d.addCallbacks(self.runCollection, self.errorStop)
def fetch(self):
''' data synchronization between RabbitMQ and OpenTSDB '''
log.debug("fetch")
def inner(driver):
try:
self.consumeDatapoints()
yield defer.succeed(None)
except:
yield defer.fail("collection failed for %s" % self.configId)
return drive(inner)
def fetch(self):
''' data synchronization between RabbitMQ and OpenTSDB '''
log.debug("fetch")
def inner(driver):
try:
messages = self.pullData()
#self.reQueue(messages)
yield defer.succeed(None)
except: yield defer.fail("collection failed for %s" % self.configId)
return drive(inner)
def connected(self):
def configTask(driver):
self.log.debug("configTask(): fetching config")
yield self.fetchConfig()
driver.next()
driveLater(self.configCycleInterval * 60, configTask)
d = drive(configTask)
d.addCallbacks(self.runCollection, self.errorStop)
def fetch(self):
''' data synchronization between Zenoss and PagerDuty '''
def inner(driver):
try:
self.sync.synchronize()
self.writeLogs()
yield defer.succeed("Synchronization succeeded for %s" % 'localhost')
except: yield defer.fail("Synchronization failed for %s" % 'localhost')
self.writeLogs()
return drive(inner)
def discoverIps(self, nets):
"""
Ping all ips, create entries in the network if necessary.
@param nets: list of networks to discover
@type nets: list
@return: successful result is a list of IPs that were added
@rtype: Twisted deferred
"""
def inner(driver):
"""
Twisted driver class to iterate through devices
@param driver: Zenoss driver
@type driver: Zenoss driver
@return: successful result is a list of IPs that were added
@rtype: Twisted deferred
"""
ips = []
goodCount = 0
# it would be nice to interleave ping/discover
for net in nets:
if self.options.subnets and len(net.children()) > 0:
continue
if not getattr(net, "zAutoDiscover", False):
self.log.info(
"Skipping network %s because zAutoDiscover is False"
% net.getNetworkName())
continue
self.log.info("Discover network '%s'", net.getNetworkName())
yield NJobs(self.options.chunkSize,
self.ping,
net.fullIpList()).start()
results = driver.next()
goodips = [
v.ipaddr for v in results if not isinstance(v, Failure)]
badips = [
v.value.ipaddr for v in results if isinstance(v, Failure)]
goodCount += len(goodips)
self.log.debug("Got %d good IPs and %d bad IPs",
len(goodips), len(badips))
yield self.config().callRemote('pingStatus',
net,
goodips,
badips,
self.options.resetPtr,
self.options.addInactive)
ips += driver.next()
self.log.info("Discovered %s active ips", goodCount)
# make sure this is the return result for the driver
yield succeed(ips)
driver.next()
d = drive(inner)
return d
def findRemoteDeviceInfo(self, ip, devicePath, deviceSnmpCommunities=None):
"""
Scan a device for ways of naming it: PTR DNS record or a SNMP name
@param ip: IP address
@type ip: string
@param devicePath: where in the DMD to put any discovered devices
@type devicePath: string
@param deviceSnmpCommunities: Optional list of SNMP community strings
to try, overriding those set on the device class
@type deviceSnmpCommunities: list
@return: result is None or a tuple containing
(community, port, version, snmp name)
@rtype: deferred: Twisted deferred
"""
from pynetsnmp.twistedsnmp import AgentProxy
def inner(driver):
"""
Twisted driver class to iterate through devices
@param driver: Zenoss driver
@type driver: Zenoss driver
@return: successful result is a list of IPs that were added
@rtype: Twisted deferred
"""
self.log.debug("findRemoteDeviceInfo.inner: Doing SNMP lookup on device %s", ip)
yield self.config().callRemote('getSnmpConfig', devicePath)
communities, ports, version, timeout, retries = driver.next()
self.log.debug("findRemoteDeviceInfo.inner: override acquired community strings")
# Override the device class communities with the ones set on
# this device, if they exist
if deviceSnmpCommunities is not None:
communities = deviceSnmpCommunities
# Reverse the communities so that ones earlier in the list have a
# higher weight.
communities.reverse()
configs = []
for i, community in enumerate(communities):
for port in ports:
port = int(port)
configs.append(SnmpV1Config(
ip, weight=i, port=port, timeout=timeout,
retries=retries, community=community))
configs.append(SnmpV2cConfig(
ip, weight=i+100, port=port, timeout=timeout,
retries=retries, community=community))
yield SnmpAgentDiscoverer().findBestConfig(configs)
driver.next()
self.log.debug("Finished SNMP lookup on device %s", ip)
return drive(inner)
def doRun(self, driver):
# test snmp connectivity
log.debug("Testing SNMP configuration")
yield self.proxy.walk('.1.3')
try:
driver.next()
except TimeoutError:
log.info("Device timed out: " + self.connInfo.summary())
if self.options.discoverCommunity:
yield self.findSnmpCommunity()
snmp_config = driver.next()
if not snmp_config:
log.error(
'Failed to rediscover the SNMP connection info for %s',
self.device.manageIp)
raise
if snmp_config.version:
self.connInfo.zSnmpVer = snmp_config.version
if snmp_config.port:
self.connInfo.zSnmpPort = snmp_config.port
if snmp_config.community:
self.connInfo.zSnmpCommunity = snmp_config.community
self.connInfo.changed = True
self.initSnmpProxy()
else:
raise
except Snmpv3Error:
log.error("Cannot connect to SNMP agent: {0}".format(
self.connInfo.summary()))
raise
except Exception:
log.exception("Unable to talk: " + self.connInfo.summary())
raise
changed = True
# FIXME: cleanup --force option #2660
if not self.options.force and self.device.snmpOid.startswith(
".1.3.6.1.4.1.9"):
yield drive(self.checkCiscoChange)
changed = driver.next()
if changed:
yield drive(self.collect)
def _connectPagerDuty(self):
''' connect to PagerDuty '''
log.info("Connecting to PagerDuty")
def inner(driver):
try:
#log.debug("using params: %s %s %s " % (self.pdhost, self.pdtoken, self.pduser))
self.sync.initPagerDuty()
self.sync.pagerduty.http.verbose = False
yield defer.succeed("Connected to PagerDuty")
except: yield defer.fail("Could not connect to PagerDuty")
self.writeLogs()
return drive(inner)
def discoverIps(self, nets):
"""
Ping all ips, create entries in the network if necessary.
@param nets: list of networks to discover
@type nets: list
@return: successful result is a list of IPs that were added
@rtype: Twisted deferred
"""
def inner(driver):
"""
Twisted driver class to iterate through devices
@param driver: Zenoss driver
@type driver: Zenoss driver
@return: successful result is a list of IPs that were added
@rtype: Twisted deferred
"""
ips = []
goodCount = 0
# it would be nice to interleave ping/discover
for net in nets:
if self.options.subnets and len(net.children()) > 0:
continue
if not getattr(net, "zAutoDiscover", False):
self.log.info(
"Skipping network %s because zAutoDiscover is False" %
net.getNetworkName())
continue
self.log.info("Discover network '%s'", net.getNetworkName())
yield NJobs(self.options.chunkSize, self.ping,
net.fullIpList()).start()
results = driver.next()
goodips = [
v.ipaddr for v in results if not isinstance(v, Failure)
]
badips = [
v.value.ipaddr for v in results if isinstance(v, Failure)
]
goodCount += len(goodips)
self.log.debug("Got %d good IPs and %d bad IPs", len(goodips),
len(badips))
yield self.config().callRemote('pingStatus', net, goodips,
badips, self.options.resetPtr,
self.options.addInactive)
ips += driver.next()
self.log.info("Discovered %s active ips", goodCount)
# make sure this is the return result for the driver
yield succeed(ips)
driver.next()
d = drive(inner)
return d
def connected( self ):
if not self.validateOptions():
self.stop()
return
self.log.info( 'Starting EmailPing. Instance: %s' % self.name )
self.configureOptions()
deferred = drive( self.configurePerformanceData )
deferred.addErrback( self.logError )
deferred.addCallback( self.startClearMailbox )
deferred.addCallback( self.startEmailCycle )
def runCollection(self, result = None):
def doCollection(driver):
self.log.debug("doCollection(): starting collection cycle")
reactor.callLater(self.options.dataCollectInterval, self.runCollection)
if not self.options.cycle:
self.stop()
if self.running:
self.log.error("last appengine collection is still running")
return
self.running = True
jobs = NJobs(200,
self.collectAppEngine,
self.datasourceMap.keys())
yield jobs.start()
driver.next()
self.log.debug("doCollection(): exiting collection cycle")
self.sendEvents(
self.rrdStats.gauge('instances',
self.options.dataCollectInterval,
len(self.datasourceMap)) +
self.rrdStats.counter('dataPoints',
self.options.dataCollectInterval,
self.rrd.dataPoints) +
self.rrdStats.gauge('cyclePoints',
self.options.dataCollectInterval,
self.rrd.endCycle())
)
def handleFinish(results):
self.running = False
for result in results:
if isinstance(result,failure.Failure):
self.log.error("handleFinish():Failure: %s"
% result)
result.printDetailedTraceback()
elif isinstance(result , Exception):
self.log.error("handleFinish():Exception: %s"
% result)
else:
self.log.debug("handleFinish(): success %s"
% result)
def handleError(error):
self.running = False
self.log.error("handleError():Error running doCollection: %s"
% error.printTraceback())
d = drive(doCollection)
d.addCallback(handleFinish)
d.addErrback(handleError)
return d
def doRun(self, driver):
# test snmp connectivity
log.debug("Testing SNMP configuration")
yield self.proxy.walk('.1.3')
try:
driver.next()
except TimeoutError:
log.info("Device timed out: " + self.connInfo.summary())
if self.options.discoverCommunity:
yield self.findSnmpCommunity()
snmp_config = driver.next()
if not snmp_config:
log.warn(
'Failed to rediscover the SNMP connection info for %s',
self.device.manageIp)
return
if snmp_config.version:
self.connInfo.zSnmpVer = snmp_config.version
if snmp_config.port:
self.connInfo.zSnmpPort = snmp_config.port
if snmp_config.community:
self.connInfo.zSnmpCommunity = snmp_config.community
self.connInfo.changed = True
self.initSnmpProxy()
else:
return
except Snmpv3Error:
log.info("Cannot connect to SNMP agent: {0}".format(self.connInfo.summary()))
return
except Exception:
log.exception("Unable to talk: " + self.connInfo.summary())
return
changed = True
# FIXME: cleanup --force option #2660
if not self.options.force and self.device.snmpOid.startswith(".1.3.6.1.4.1.9"):
yield drive(self.checkCiscoChange)
changed = driver.next()
if changed:
yield drive(self.collect)
def _connectZenoss(self):
''' connect to Zenoss '''
log.info("Connecting to Zenoss")
def inner(driver):
try:
#log.debug("using params: %s %s %s" % (self.zenhost, self.zenuser, self.zenpass))
self.sync.initZenoss()
self.sync.zenoss.http.verbose = False
yield defer.succeed("Connected to Zenoss")
except:
yield defer.fail("Could not connect to Zenoss")
self.writeLogs()
return drive(inner)
def main(self, unused=None):
"""
Wrapper around the mainLoop
@param unused: unused (unused)
@type unused: string
@return: Twisted deferred object
@rtype: Twisted deferred object
"""
self.finished = []
d = drive(self.mainLoop)
d.addCallback(self.timeoutClients)
return d
def timeoutClients(self, unused=None):
"""
Check to see which clients have timed out and which ones haven't.
Stop processing anything that's timed out.
@param unused: unused (unused)
@type unused: string
"""
reactor.callLater(1, self.timeoutClients)
self._timeoutClients()
d = drive(self.fillCollectionSlots)
d.addCallback(self.checkStop)
d.addErrback(self.fillError)
def timeoutClients(self, unused=None):
"""
Check to see which clients have timed out and which ones haven't.
Stop processing anything that's timed out.
@param unused: unused (unused)
@type unused: string
"""
reactor.callLater(1, self.timeoutClients)
self._timeoutClients()
d = drive(self.fillCollectionSlots)
d.addCallback(self.checkStop)
d.addErrback(self.fillError)
def main(self, unused=None):
"""
Wrapper around the mainLoop
@param unused: unused (unused)
@type unused: string
@return: Twisted deferred object
@rtype: Twisted deferred object
"""
self.finished = []
d = drive(self.mainLoop)
d.addCallback(self.timeoutClients)
return d
def configure(self):
"""
"""
def inner(driver):
#log.info("Finding connection for %s",self._devId)
if self.siebelClient is not None:
basemsg = self.connectionInfo(self.siebelClient)
# this is a good connection
if self.siebelClient.connected is True and self.siebelClient.blocked is False:
yield defer.succeed("%s using connection to %s" % (self._devId, basemsg))
else:
yield defer.fail("%s could not find connection to %s" % (self._devId, basemsg))
return drive(inner)
def configure(self):
"""
Get our configuration from zenhub
"""
# add in the code to fetch cycle time, etc.
self.log.info("Getting configuration from ZenHub...")
def inner(driver):
"""
Generator function to gather our configuration
@param driver: driver object
@type driver: driver object
"""
self.log.debug('fetching monitor properties')
yield self.config().callRemote('propertyItems')
items = dict(driver.next())
# If the cycletime option is not specified or zero, then use the
# modelerCycleInterval value in the database.
if not self.options.cycletime:
self.modelerCycleInterval = items.get('modelerCycleInterval',
_DEFAULT_CYCLE_INTERVAL)
self.configCycleInterval = items.get('configCycleInterval',
self.configCycleInterval)
reactor.callLater(self.configCycleInterval * 60, self.configure)
self.log.debug("Getting threshold classes...")
yield self.config().callRemote('getThresholdClasses')
self.remote_updateThresholdClasses(driver.next())
self.log.debug("Getting collector thresholds...")
yield self.config().callRemote('getCollectorThresholds')
thresholds = driver.next()
threshold_notifier = ThresholdNotifier(self.sendEvent, thresholds)
self.rrdStats.config(self.name,
self.options.monitor,
self.metricWriter(),
threshold_notifier,
self.derivativeTracker())
self.log.debug("Getting collector plugins for each DeviceClass")
yield self.config().callRemote('getClassCollectorPlugins')
self.classCollectorPlugins = driver.next()
self.configLoaded = True
return drive(inner)
def configure(self):
"""
Get our configuration from zenhub
"""
# add in the code to fetch cycle time, etc.
self.log.info("Getting configuration from ZenHub...")
def inner(driver):
"""
Generator function to gather our configuration
@param driver: driver object
@type driver: driver object
"""
self.log.debug('fetching monitor properties')
yield self.config().callRemote('propertyItems')
items = dict(driver.next())
# If the cycletime option is not specified or zero, then use the
# modelerCycleInterval value in the database.
if not self.options.cycletime:
self.modelerCycleInterval = items.get('modelerCycleInterval',
_DEFAULT_CYCLE_INTERVAL)
self.configCycleInterval = items.get('configCycleInterval',
self.configCycleInterval)
reactor.callLater(self.configCycleInterval * 60, self.configure)
self.log.debug("Getting threshold classes...")
yield self.config().callRemote('getThresholdClasses')
self.remote_updateThresholdClasses(driver.next())
self.log.debug("Getting collector thresholds...")
yield self.config().callRemote('getCollectorThresholds')
thresholds = driver.next()
threshold_notifier = ThresholdNotifier(self.sendEvent, thresholds)
self.rrdStats.config(self.name, self.options.monitor,
self.metricWriter(), threshold_notifier,
self.derivativeTracker())
self.log.debug("Getting collector plugins for each DeviceClass")
yield self.config().callRemote('getClassCollectorPlugins')
self.classCollectorPlugins = driver.next()
self.configLoaded = True
return drive(inner)
def configure(self):
"""
Get our configuration from zenhub
"""
# add in the code to fetch cycle time, etc.
def inner(driver):
"""
Generator function to gather our configuration
@param driver: driver object
@type driver: driver object
"""
self.log.debug('fetching monitor properties')
yield self.config().callRemote('propertyItems')
items = dict(driver.next())
# If the cycletime option is not specified or zero, then use the
# modelerCycleInterval value in the database.
if not self.options.cycletime:
self.modelerCycleInterval = items.get('modelerCycleInterval',
_DEFAULT_CYCLE_INTERVAL)
self.configCycleInterval = items.get('configCycleInterval',
self.configCycleInterval)
reactor.callLater(self.configCycleInterval * 60, self.configure)
self.log.debug("Getting threshold classes...")
yield self.config().callRemote('getThresholdClasses')
self.remote_updateThresholdClasses(driver.next())
self.log.debug("Fetching default RRDCreateCommand...")
yield self.config().callRemote('getDefaultRRDCreateCommand')
createCommand = driver.next()
self.log.debug("Getting collector thresholds...")
yield self.config().callRemote('getCollectorThresholds')
self.rrdStats.config(self.options.monitor,
self.name,
driver.next(),
createCommand)
self.log.debug("Getting collector plugins for each DeviceClass")
yield self.config().callRemote('getClassCollectorPlugins')
self.classCollectorPlugins = driver.next()
return drive(inner)
def findSnmpCommunity(self):
def inner(driver):
"""
Twisted driver class to iterate through devices
@param driver: Zenoss driver
@type driver: Zenoss driver
@return: successful result is a list of IPs that were added
@rtype: Twisted deferred
"""
log.info("Rediscovering SNMP connection info for %s",
self.device.id)
communities = list(self.device.zSnmpCommunities)
communities.reverse()
ports = self.device.zSnmpDiscoveryPorts
ports = ports if ports else [self.device.zSnmpPort]
configs = []
weight = 0
for community in communities:
for port in ports:
weight += 1
port = int(port)
configs.append(
SnmpV1Config(self.device.manageIp,
weight=weight,
port=port,
timeout=self.connInfo.zSnmpTimeout,
retries=self.connInfo.zSnmpTries,
community=community))
configs.append(
SnmpV2cConfig(self.device.manageIp,
weight=weight + 1000,
port=port,
timeout=self.connInfo.zSnmpTimeout,
retries=self.connInfo.zSnmpTries,
community=community))
yield SnmpAgentDiscoverer().findBestConfig(configs)
driver.next()
return drive(inner)
def runCollection(self, result=None):
def doCollection(driver):
self.log.debug("doCollection(): starting collection cycle")
reactor.callLater(self.options.dataCollectInterval,
self.runCollection)
if not self.options.cycle:
self.stop()
if self.running:
self.log.error("last appengine collection is still running")
return
self.running = True
jobs = NJobs(200, self.collectAppEngine, self.datasourceMap.keys())
yield jobs.start()
driver.next()
self.log.debug("doCollection(): exiting collection cycle")
self.sendEvents(
self.rrdStats.gauge(
'instances', self.options.dataCollectInterval,
len(self.datasourceMap)) + self.rrdStats.counter(
'dataPoints', self.options.dataCollectInterval,
self.rrd.dataPoints) +
self.rrdStats.gauge('cyclePoints', self.options.
dataCollectInterval, self.rrd.endCycle()))
def handleFinish(results):
self.running = False
for result in results:
if isinstance(result, failure.Failure):
self.log.error("handleFinish():Failure: %s" % result)
result.printDetailedTraceback()
elif isinstance(result, Exception):
self.log.error("handleFinish():Exception: %s" % result)
else:
self.log.debug("handleFinish(): success %s" % result)
def handleError(error):
self.running = False
self.log.error("handleError():Error running doCollection: %s" %
error.printTraceback())
d = drive(doCollection)
d.addCallback(handleFinish)
d.addErrback(handleError)
return d
def testDevice(device, datasource):
log.info("Testing mail transaction against device %s" % (device, ))
def go(driver):
from Products.ZenUtils.ZenScriptBase import ZenScriptBase
from ZenPacks.zenoss.ZenMailTx.MailTxConfigService import MailTxConfigService
zendmd = ZenScriptBase(noopts=True, connect=True)
dmd = zendmd.dmd
d = dmd.Devices.findDevice(device)
if not d:
sys.stderr.write("Unable to find device %s\n" % device)
sys.exit(1)
log.setLevel(logging.DEBUG)
service = MailTxConfigService(dmd, d.perfServer().id)
if not service:
sys.stderr.write("Unable to find configuration for %s" % device)
proxy = service.remote_getDeviceConfigs([device])
if proxy:
proxy = proxy[0]
else:
raise ValueError(
"Unable to find a valid MailTx config for device %s" % device)
config = proxy.datasources
if datasource:
config = [c for c in proxy.datasources if c.name == datasource]
if not config:
raise ValueError(
"Unable to find a MailTx config %s for device %s" %
(datasource or '', device))
config = config[0]
config.ignoreIds = set()
now = time.time()
yield sendMessage(config)
log.debug("Result of message send: %s", driver.next())
yield getMessage(config, 5.0)
log.debug("Result of message fetch: %s", driver.next())
log.info("Message delivered in %.2f seconds" % (time.time() - now))
d = drive(go)
d.addErrback(error)
d.addCallback(stop)
reactor.run()
def configure(self):
"""
Get our configuration from zenhub
"""
# add in the code to fetch cycle time, etc.
def inner(driver):
"""
Generator function to gather our configuration
@param driver: driver object
@type driver: driver object
"""
self.log.debug('fetching monitor properties')
yield self.config().callRemote('propertyItems')
items = dict(driver.next())
# If the cycletime option is not specified or zero, then use the
# modelerCycleInterval value in the database.
if not self.options.cycletime:
self.modelerCycleInterval = items.get('modelerCycleInterval',
_DEFAULT_CYCLE_INTERVAL)
self.configCycleInterval = items.get('configCycleInterval',
self.configCycleInterval)
reactor.callLater(self.configCycleInterval * 60, self.configure)
self.log.debug("Getting threshold classes...")
yield self.config().callRemote('getThresholdClasses')
self.remote_updateThresholdClasses(driver.next())
self.log.debug("Fetching default RRDCreateCommand...")
yield self.config().callRemote('getDefaultRRDCreateCommand')
createCommand = driver.next()
self.log.debug("Getting collector thresholds...")
yield self.config().callRemote('getCollectorThresholds')
self.rrdStats.config(self.options.monitor, self.name,
driver.next(), createCommand)
self.log.debug("Getting collector plugins for each DeviceClass")
yield self.config().callRemote('getClassCollectorPlugins')
self.classCollectorPlugins = driver.next()
return drive(inner)
def fetchConfig(self):
"""
Get configuration values from zenhub
@return: Twisted deferred
@rtype: Twisted deferred
"""
def doFetchConfig(driver):
now = time.time()
yield self.model().callRemote('getDefaultRRDCreateCommand')
createCommand = driver.next()
yield self.model().callRemote('getZenProcessParallelJobs')
self.parallelJobs = int(driver.next())
yield self.model().callRemote('propertyItems')
self.setPropertyItems(driver.next())
self.rrd = RRDUtil(createCommand, self.processCycleInterval)
yield self.model().callRemote('getThresholdClasses')
self.remote_updateThresholdClasses(driver.next())
yield self.model().callRemote('getCollectorThresholds')
self.rrdStats.config(self.options.monitor,
self.name,
driver.next(),
createCommand)
devices = []
if self.options.device:
devices = [self.options.device]
yield self.model().callRemote('getSunMibProcessConf', devices)
driver.next()
self.sendEvents(
self.rrdStats.gauge('configTime',
self.processConfigInterval,
time.time() - now)
)
return drive(doFetchConfig)
def oneDevice(self, device):
"""
Contact one device and return a deferred which gathers data from
the device.
@parameter device: proxy object to the remote computer
@type device: Device object
@return: job to scan a device
@rtype: Twisted deferred object
"""
def go(driver):
"""
Generator object to gather information from a device.
"""
try:
device.open()
yield self.scanDevice(device)
driver.next()
# Only fetch performance data if status data was found.
if device.snmpStatus == 0:
yield self.fetchPerf(device)
driver.next()
else:
log.warn("Failed to find performance data for %s",
device.name)
except:
log.debug('Failed to scan device %s' % device.name)
def close(res):
"""
Twisted closeBack and errBack function which closes any
open connections.
"""
try:
device.close()
except:
log.debug("Failed to close device %s" % device.name)
d = drive(go)
d.addBoth(close)
return d
def mainLoop(self, driver):
"""
Main collection loop, a Python iterable
@param driver: driver object
@type driver: driver object
@return: Twisted deferred object
@rtype: Twisted deferred object
"""
if self.options.cycle:
self.isMainScheduled = True
driveLater(self.cycleTime(), self.mainLoop)
if self.clients:
self.log.error("Modeling cycle taking too long")
return
# ZEN-26637 - did we collect during collector loop?
self.didCollect = False
self.mainLoopGotDeviceList = False
self.start = time.time()
self.collectorLoopIteration = self.collectorLoopIteration + 1
self.log.info("Starting collector loop #{:03d}...".format(
self.collectorLoopIteration))
yield self.getDeviceList()
deviceList = driver.next()
self.log.debug("getDeviceList returned %s devices", len(deviceList))
self.log.debug("getDeviceList returned %s devices", deviceList)
self.devicegen = iter(deviceList)
self.iterationDeviceCount = len(deviceList)
self.processedDevicesCount = 0
self.log.info(
"Got %d devices to be scanned during collector loop #%03d",
self.iterationDeviceCount, self.collectorLoopIteration)
d = drive(self.fillCollectionSlots)
d.addErrback(self.fillError)
self.mainLoopGotDeviceList = True
yield d
driver.next()
self.log.debug("Collection slots filled")
def discoverRouters(self, rootdev, seenips=None):
"""
Discover all default routers based on DMD configuration.
@param rootdev: device root in DMD
@type rootdev: device class
@param seenips: list of IP addresses
@type seenips: list of strings
@return: Twisted/Zenoss Python iterable
@rtype: Python iterable
"""
if not seenips:
seenips = []
def inner(driver):
"""
Twisted driver class to iterate through devices
@param driver: Zenoss driver
@type driver: Zenoss driver
@return: successful result is a list of IPs that were added
@rtype: Twisted deferred
"""
yield self.config().callRemote('followNextHopIps', rootdev.id)
for ip in driver.next():
if ip in seenips:
continue
self.log.info("device '%s' next hop '%s'", rootdev.id, ip)
seenips.append(ip)
yield self.discoverDevice(ip, devicepath="/Network/Router")
router = driver.next()
if not router:
continue
yield self.discoverRouters(router, seenips)
driver.next()
return drive(inner)
def configure(self):
def inner(driver):
self.log.info("fetching default RRDCreateCommand")
yield self.model().callRemote('getDefaultRRDCreateCommand')
createCommand = driver.next()
self.log.info("getting threshold classes")
yield self.model().callRemote('getThresholdClasses')
self.remote_updateThresholdClasses(driver.next())
self.log.info("getting collector thresholds")
yield self.model().callRemote('getCollectorThresholds')
self.rrdStats.config(self.options.monitor, self.name,
driver.next(), createCommand)
self.heartbeat()
self.reportCycle()
d = drive(inner)
def error(result):
self.log.error("Unexpected error in configure: %s" % result)
d.addErrback(error)
return d
def test(self, service):
def Test(driver):
yield service.callRemote('raiseConflictError', "an error message")
drive(Test).addBoth(self.complete)
self.log.debug("Client %s finished" % device.id)
try:
self.clients.remove(collectorClient)
self.finished.append(collectorClient)
except ValueError:
self.log.debug(
"Client %s not found in in the list"
" of active clients", device.id)
self.log.info(
"Finished processing client within collector loop #{0:03d}".
format(self.collectorLoopIteration))
d = drive(self.fillCollectionSlots)
d.addErrback(self.fillError)
d = drive(processClient)
d.addBoth(processClientFinished)
def savePluginData(self, deviceName, pluginName, dataType, data):
filename = "/tmp/%s.%s.%s.pickle.gz" % (deviceName, pluginName,
dataType)
try:
with gzip.open(filename, 'wb') as fd:
pickle.dump(data, fd)
except Exception as ex:
self.log.warn("Unable to save data into file '%s': %s", filename,
ex)
def fillError(self, reason):
"""
Twisted errback routine to log an error when
