def _find_instances_in_catalog(self, count=False):
model_catalog = IModelCatalogTool(self.dmd)
query = {"productClassId": self.idx_uid()}
if count:
return model_catalog.search(query=query, limit=0)
else:
return model_catalog.search(query=query)
def testDeleteOrganizerRemovesDevices(self):
"""When we delete an organizer all the devices assigned to it should not
still have a relationship to it in the catalog
"""
# Create Organizer
organizer = self.facade.addOrganizer("/zport/dmd/Groups",
'testOrganizer')
organizer_path = organizer.uid
catalog = IModelCatalogTool(self.dmd)
# Add some devices to it (use createInstance to create a device)
devices = self.dmd.Devices
test_device = devices.createInstance('testDevice')
self.facade.moveDevices(['/'.join(test_device.getPhysicalPath())],
organizer_path)
organizer = self.dmd.unrestrictedTraverse(organizer_path)
# test we have added the device
self.assertEqual(len(organizer.getDevices()), 1,
"make sure we saved our device")
deviceBrains = catalog.search(
paths='/'.join(organizer.getPhysicalPath()))
self.assertTrue(
deviceBrains.total > 1,
" At this point we should have the organizer and the device")
# Delete the Organizer
self.facade.deleteNode(organizer_path)
# Get the devices directly from the path
deviceBrains = catalog.search(
paths='/'.join(organizer.getPhysicalPath()))
self.assertEqual(deviceBrains.total, 0,
" we should not have any devices at this point")
def testDeleteOrganizerRemovesDevices(self):
"""When we delete an organizer all the devices assigned to it should not
still have a relationship to it in the catalog
"""
# Create Organizer
organizer = self.facade.addOrganizer("/zport/dmd/Groups", 'testOrganizer')
organizer_path = organizer.uid
catalog = IModelCatalogTool(self.dmd)
# Add some devices to it (use createInstance to create a device)
devices = self.dmd.Devices
test_device = devices.createInstance('testDevice')
self.facade.moveDevices(['/'.join(test_device.getPhysicalPath())], organizer_path)
organizer = self.dmd.unrestrictedTraverse(organizer_path)
# test we have added the device
self.assertEqual(len(organizer.getDevices()), 1, "make sure we saved our device")
deviceBrains = catalog.search(paths='/'.join(organizer.getPhysicalPath()))
self.assertTrue(deviceBrains.total > 1, " At this point we should have the organizer and the device")
# Delete the Organizer
self.facade.deleteNode(organizer_path)
# Get the devices directly from the path
deviceBrains = catalog.search(paths='/'.join(organizer.getPhysicalPath()))
self.assertEqual(deviceBrains.total, 0, " we should not have any devices at this point")
def _find_instances_in_catalog(self, count=False):
model_catalog = IModelCatalogTool(self.dmd)
query = { "productClassId": self.idx_uid() }
if count:
return model_catalog.search(query=query, limit=0)
else:
return model_catalog.search(query=query)
def _search_catalog(self, obj, types=(), paths=(), query=None):
if USE_MODEL_CATALOG:
catalog = IModelCatalogTool(obj)
else:
catalog = ICatalogTool(obj)
return catalog.search(types=types, paths=paths, query=query)
def validate_devices_in_networks(self):
"""
Get deviceId's of every network using both catalogs and compare results
Network -> Devices
"""
networks = get_networks(dmd)
layer3_catalog = dmd.ZenLinkManager.layer3_catalog
model_catalog = IModelCatalogTool(dmd)
failed_networks = {}
for network in networks:
# Devices under device class in global catalog
query = Eq('networkId', network)
layer3_brains = layer3_catalog.evalAdvancedQuery(query)
layer3_device_ids = set([ brain.deviceId for brain in layer3_brains if brain.deviceId ])
model_catalog_brains = model_catalog.search(query=query)
model_catalog_device_ids = set([ brain.deviceId.split("/")[-1] for brain in model_catalog_brains.results if brain.deviceId ])
if not len(layer3_device_ids - model_catalog_device_ids) == len(model_catalog_device_ids - layer3_device_ids) == 0:
#import pdb; pdb.set_trace()
failed_networks[network] = (layer3_device_ids, model_catalog_device_ids)
if failed_networks:
print "TEST FAILED: Catalogs return different devices for the following networks:"
print "\t\t{0}".format(failed_networks.keys())
else:
print "TEST PASSED: Both catalogs returned the same devices for all networks."
return len(failed_networks) == 0
class ModelCatalogScanInfo(CatalogScanInfo):
def __init__(self, dmd):
super(ModelCatalogScanInfo, self).__init__(dmd, 'model_catalog', '')
self.updates = []
if USE_MODEL_CATALOG:
self._catalog_tool = IModelCatalogTool(self.dmd)
def size(self):
search_results = self._catalog_tool.search(filterPermissions=False,
start=0,
limit=0)
return search_results.total
def get_brains(self):
return self._catalog_tool.cursor_search()
def uncatalog_object(self, uid):
self.updates.append(IndexUpdate(None, op=UNINDEX, uid=uid))
if len(self.updates) % 1000 == 0:
self.commit()
def exists(self):
return USE_MODEL_CATALOG
def commit(self):
if self.updates:
self._catalog_tool.model_index.process_batched_updates(
self.updates)
self.updates = []
def test_device_classes_devices(self):
""" Check devices under device classes are the same """
failed_device_classes = []
for dc in DEVICE_CLASSES:
dc_object = dmd.unrestrictedTraverse(dc)
# Devices under device class in global catalog
global_catalog = ICatalogTool(dc_object)
global_catalog_brains = global_catalog.search('Products.ZenModel.Device.Device')
global_catalog_results = set([ brain.getPath() for brain in global_catalog_brains.results ])
# Devices under device class in model catalog
model_catalog = IModelCatalogTool(dc_object)
model_catalog_brains = model_catalog.search('Products.ZenModel.Device.Device', limit=10000)
model_catalog_results = set([ brain.getPath() for brain in model_catalog_brains.results ])
result = "FAILED"
if len(global_catalog_results - model_catalog_results) == 0 and \
len(model_catalog_results-global_catalog_results) ==0:
result = "PASSED"
else:
failed_device_classes.append(dc)
if not failed_device_classes:
print "TEST PASSED: All devices found in the same device classes for both catalogs!!"
else:
print "TEST FAILED: The following device classes have different devices in the catalogs:"
for failed in failed_device_classes:
print "\t{0}".format(failed)
return len(failed_device_classes) == 0
def moveProcess(self, uid, targetUid):
obj = self._getObject(uid)
target = self._getObject(targetUid)
brainsCollection = []
# reindex all the devices and processes underneath this guy and the target
for org in (obj.getPrimaryParent().getPrimaryParent(), target):
catalog = IModelCatalogTool(org)
brainsCollection.append(catalog.search(OSProcess))
if isinstance(obj, OSProcessClass):
source = obj.osProcessOrganizer()
source.moveOSProcessClasses(targetUid, obj.id)
newObj = getattr(target.osProcessClasses, obj.id)
elif isinstance(obj, OSProcessOrganizer):
source = aq_parent(obj)
source.moveOrganizer(targetUid, (obj.id, ))
newObj = getattr(target, obj.id)
else:
raise Exception('Illegal type %s' % obj.__class__.__name__)
# fire the object moved event for the process instances (will update catalog)
for brains in brainsCollection:
objs = imap(unbrain, brains)
for item in objs:
notify(
ObjectMovedEvent(item, item.os(), item.id, item.os(),
item.id))
return newObj.getPrimaryPath()
def _getAdvancedQueryDeviceList(self,
offset=0,
count=50,
filter='',
orderby='name',
orderdir='asc'):
"""
Ask the catalog for devices matching the criteria specified.
"""
context = self.context
if not isinstance(context, DeviceOrganizer):
context = self.context.dmd.Devices
catalog = IModelCatalogTool(context).devices
devfilter = '(?is).*%s.*' % filter
filterquery = Or(MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter))
query = Eq('uid', context.absolute_url_path()) & filterquery
objects = catalog.search(query=query,
start=int(offset),
limit=int(count),
order_by=orderby,
reverse=orderdir != 'asc')
objects = list(objects)
totalCount = len(objects)
return totalCount, objects
def _getAdvancedQueryDeviceList(self, offset=0, count=50, filter='',
orderby='name', orderdir='asc'):
"""
Ask the catalog for devices matching the criteria specified.
"""
context = self.context
if not isinstance(context, DeviceOrganizer):
context = self.context.dmd.Devices
catalog = IModelCatalogTool(context).devices
devfilter = '(?is).*%s.*' % filter
filterquery = Or(
MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter)
)
query = Eq('uid', context.absolute_url_path()
) & filterquery
objects = catalog.search(
query=query,
start=int(offset),
limit=int(count),
order_by=orderby,
reverse=orderdir != 'asc'
)
objects = list(objects)
totalCount = len(objects)
return totalCount, objects
def validate_devices_interfaces_and_mac_addresses(self):
"""
get macs of avery device in the system with both catalogs
"""
layer2_catalog = dmd.ZenLinkManager.layer2_catalog
model_catalog = IModelCatalogTool(dmd)
failed_devices = []
for device in get_all_devices(dmd):
device_uid = "/".join(device.getPrimaryPath())
layer2_query = Eq('deviceId', device_uid)
layer2_brains = layer2_catalog.evalAdvancedQuery(layer2_query)
layer2_macs = set([ brain.macaddress for brain in layer2_brains if brain.macaddress ])
layer2_ifaces = set([ brain.interfaceId for brain in layer2_brains if brain.interfaceId ])
model_catalog_query = And(Eq('deviceId', "{0}".format(device_uid)), Eq('meta_type', "IpInterface"))
search_results = model_catalog.search(query=model_catalog_query)
model_catalog_brains = [ brain for brain in search_results.results ]
model_catalog_macs = set([ brain.macaddress for brain in model_catalog_brains if brain.macaddress ])
model_catalog_ifaces = set([ brain.interfaceId for brain in model_catalog_brains if brain.interfaceId ])
if not ( len(layer2_macs - model_catalog_macs) == len(model_catalog_macs- layer2_macs) == 0 and \
len(layer2_ifaces - model_catalog_ifaces) == len(model_catalog_ifaces- layer2_ifaces) == 0 ):
failed_devices.append(device)
if failed_devices:
print "TEST FAILED: Catalogs return different mac addresses or interfaces for the following devices:"
for dev in failed_devices:
print "\t\t{0}".format(dev.getPrimaryId())
else:
print "TEST PASSED: Both catalogs returned the same mac addresses and interfaces for every device."
return len(failed_devices) == 0
def _getDeviceBatch(self,
selectstatus='none',
goodevids=[],
badevids=[],
offset=0,
count=50,
filter='',
orderby='titleOrId',
orderdir='asc'):
unused(count, offset, orderby, orderdir)
if not isinstance(goodevids, (list, tuple)):
goodevids = [goodevids]
if not isinstance(badevids, (list, tuple)):
badevids = [badevids]
if selectstatus == 'all':
idquery = ~In('id', badevids)
else:
idquery = In('id', goodevids)
devfilter = '(?is).*%s.*' % filter
filterquery = Or(MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter))
query = Eq('uid', self.context.absolute_url_path()) & idquery
query = query & filterquery
catalog = IModelCatalogTool(self.context)
objects = catalog.search(query=query)
return [x['id'] for x in objects]
def getIpAddresses(self, limit=0, start=0, sort='ipAddressAsInt', dir='DESC',
params=None, uid=None, criteria=()):
infos = []
cat = IModelCatalogTool(self._getObject(uid))
reverse = dir=='DESC'
brains = cat.search("Products.ZenModel.IpAddress.IpAddress",
start=start, limit=limit,
orderby=sort, reverse=reverse)
for brain in brains:
infos.append(IInfo(unbrain(brain)))
devuuids = set(info.device.uuid for info in infos if info.device)
# get ping severities
zep = getFacade('zep')
pingSeverities = zep.getEventSeverities(devuuids,
severities=(),
status=(),
eventClass=Status_Ping)
self._assignPingStatuses(infos, pingSeverities)
# get snmp severities
snmpSeverities = zep.getEventSeverities(devuuids,
severities=(),
status=(),
eventClass=Status_Snmp)
self._assignSnmpStatuses(infos, snmpSeverities)
return SearchResults(infos, brains.total, brains.hash_)
def validate_ips_and_networks_for_device(self):
"""
for every device, get the ips and networks
Device -> Ip Addresses
Device -> Network
"""
layer3_catalog = dmd.ZenLinkManager.layer3_catalog
model_catalog = IModelCatalogTool(dmd)
failed_devices = []
for device in get_all_devices(dmd):
l3_query = Eq('deviceId', device.id)
layer3_brains = layer3_catalog.evalAdvancedQuery(l3_query)
layer3_ips = set([ brain.ipAddressId for brain in layer3_brains if brain.ipAddressId ])
layer3_nets = set([ brain.networkId for brain in layer3_brains if brain.networkId ])
model_catalog_query = And(Eq('deviceId', "*{0}".format(device.id)), Eq('meta_type', "IpAddress"))
search_results = model_catalog.search(query=model_catalog_query)
model_catalog_brains = [ brain for brain in search_results.results ]
model_catalog_ips = set([ brain.ipAddressId for brain in model_catalog_brains if brain.ipAddressId ])
model_catalog_nets = set([ brain.networkId for brain in model_catalog_brains if brain.networkId ])
if not (len(layer3_ips - model_catalog_ips) == len(model_catalog_ips- layer3_ips) == 0 and \
len(layer3_nets - model_catalog_nets) == len(model_catalog_nets- layer3_nets) == 0) :
failed_devices.append(device)
if failed_devices:
print "TEST FAILED: Catalogs return different ip addresses for the following devices:"
for dev in failed_devices:
print "\t\t{0}".format(dev.getPrimaryId())
else:
print "TEST PASSED: Both catalogs returned the same ips for every device."
return len(failed_devices) == 0
def getAddTemplateTargets(self, query=None):
"""
@returns list of targets for our new template
"""
cat = IModelCatalogTool(self._dmd)
results = []
# it can be any device class that we add the template too
brains = cat.search(types=[DeviceClass])
for brain in brains:
# HACK: use the path to get the organizer name so we don't have to wake up the object
label = brain.getPath()
if label == "/zport/dmd/Devices":
label = "/"
else:
label = label.replace("/zport/dmd/Devices/", "/")
results.append(dict(uid=brain.getPath(), label=label))
# the display is organizer name and we do not index it
# so filter after the query
if query is not None:
results = [
result for result in results
if query.lower() in result['label'].lower()
]
return sorted(results, key=lambda org: org['label'])
def cutover(self, dmd):
log.info("Updating relationships on Ip Addresses")
catalog = IModelCatalogTool(dmd.Networks)
for ip in catalog.search(IpAddress):
try:
ip.getObject().buildRelations()
#some objects can fail relations building process
except (Exception, ):
continue
log.info("Updating relationships on Devices")
for dev in dmd.Devices.getSubDevices():
try:
dev.buildRelations()
if dev.manageIp:
ipobj = dev.getNetworkRoot().findIp(dev.manageIp)
if ipobj:
dev.ipaddress.addRelation(ipobj)
else:
ipWithoutNetmask, netmask = ipAndMaskFromIpMask(
dev.manageIp)
ipobj = dev.getNetworkRoot().createIp(
ipWithoutNetmask, netmask)
dev.ipaddress.addRelation(ipobj)
notify(IndexingEvent(ipobj))
#some objects can fail relations building process
except (Exception, ):
continue
def moveProcess(self, uid, targetUid):
obj = self._getObject(uid)
target = self._getObject(targetUid)
brainsCollection = []
# reindex all the devices and processes underneath this guy and the target
for org in (obj.getPrimaryParent().getPrimaryParent(), target):
catalog = IModelCatalogTool(org)
brainsCollection.append(catalog.search(OSProcess))
if isinstance(obj, OSProcessClass):
source = obj.osProcessOrganizer()
source.moveOSProcessClasses(targetUid, obj.id)
newObj = getattr(target.osProcessClasses, obj.id)
elif isinstance(obj, OSProcessOrganizer):
source = aq_parent(obj)
source.moveOrganizer(targetUid, (obj.id,))
newObj = getattr(target, obj.id)
else:
raise Exception('Illegal type %s' % obj.__class__.__name__)
# fire the object moved event for the process instances (will update catalog)
for brains in brainsCollection:
objs = imap(unbrain, brains)
for item in objs:
notify(ObjectMovedEvent(item, item.os(), item.id, item.os(), item.id))
return newObj.getPrimaryPath()
def _findDevices(self, identifier, ipAddress, limit=None):
"""
Returns a tuple ([device brains], [devices]) searching manage IP and
interface IPs. limit is the maximum total number in both lists.
"""
dev_cat = IModelCatalogTool(self._devices)
try:
ip_address = next(i for i in (ipAddress, identifier) if isip(i))
ip_decimal = ipToDecimal(ip_address)
except Exception:
ip_address = None
ip_decimal = None
quoted_id = quote_and_escape(identifier)
query_set = Or(Eq('id', quoted_id), Eq('name', quoted_id))
if ip_decimal is not None:
query_set.addSubquery(Eq('decimal_ipAddress', str(ip_decimal)))
device_brains = list(
dev_cat.search(types=Device,
query=query_set,
limit=limit,
filterPermissions=False,
fields=["uid", "uuid"]))
if device_brains:
return device_brains, []
if ip_decimal is not None:
# don't search interfaces for 127.x.x.x IPv4 addresses
if ipToDecimal('126.255.255.255') < ip_decimal < ipToDecimal(
'128.0.0.0'):
ip_decimal = None
# don't search interfaces for the ::1 IPv6 address
elif ipToDecimal('::1') == ip_decimal:
ip_decimal = None
if ip_decimal is None:
return [], []
net_cat = IModelCatalogTool(self._networks)
results = net_cat.search(types=IpAddress,
query=(Eq('name', ip_address)),
limit=limit,
filterPermissions=False)
devices = [brain.getObject().device() for brain in results]
return device_brains, devices
def _search_ip_in_catalog(self, ip):
""" Return list of brains that match ip """
cat = IModelCatalogTool(self.getNetworkRoot())
query = {}
query["objectImplements"] = "Products.ZenModel.IpAddress.IpAddress"
query["id"] = ipwrap(ip)
search_result = cat.search(query=query)
return [brain for brain in search_result.results]
def _findDevices(self, identifier, ipAddress, limit=None):
"""
Returns a tuple ([device brains], [devices]) searching manage IP and
interface IPs. limit is the maximum total number in both lists.
"""
dev_cat = IModelCatalogTool(self._devices)
try:
ip_address = next(i for i in (ipAddress, identifier) if isip(i))
ip_decimal = ipToDecimal(ip_address)
except Exception:
ip_address = None
ip_decimal = None
quoted_id = quote_and_escape(identifier)
query_set = Or(Eq('id', quoted_id), Eq('name', quoted_id))
if ip_decimal is not None:
query_set.addSubquery(Eq('decimal_ipAddress', str(ip_decimal)))
device_brains = list(dev_cat.search(types=Device,
query=query_set,
limit=limit,
filterPermissions=False,
fields=["uid", "uuid"]))
if device_brains:
return device_brains, []
if ip_decimal is not None:
# don't search interfaces for 127.x.x.x IPv4 addresses
if ipToDecimal('126.255.255.255') < ip_decimal < ipToDecimal('128.0.0.0'):
ip_decimal = None
# don't search interfaces for the ::1 IPv6 address
elif ipToDecimal('::1') == ip_decimal:
ip_decimal = None
if ip_decimal is None:
return [], []
net_cat = IModelCatalogTool(self._networks)
results = net_cat.search(types=IpAddress,
query=(Eq('name', ip_address)),
limit = limit,
filterPermissions = False)
devices = [brain.getObject().device() for brain in results]
return device_brains, devices
def _get_devices_under_path(self, path):
""" Returnd device's brains """
model_catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}".format(path)
fields = ["id", "path"]
result = model_catalog.search(query=query, fields=fields)
return result.results
def _get_devices_under_path(self, path):
""" Returnd device's brains """
model_catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}".format(path)
fields = ["id", "path"]
result = model_catalog.search(query=query, fields=fields)
return result.results
def _getSubdevices(self, brains=False):
""" Helper method to search for devices/devices brains under the current organizer """
catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}*".format("/".join(self.getPhysicalPath()))
if not brains:
return getObjectsFromModelCatalog(catalog, query, LOG)
else:
return catalog.search(query=query).results
def _getSubdevices(self, brains=False):
""" Helper method to search for devices/devices brains under the current organizer """
catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}*".format("/".join(self.getPhysicalPath()))
if not brains:
return getObjectsFromModelCatalog(catalog, query, LOG)
else:
return catalog.search(query=query).results
def _get_all_devices(self):
""" """
model_catalog = IModelCatalogTool(dmd)
query = {}
query['objectImplements'] = "Products.ZenModel.Device.Device"
search_results = model_catalog.search(query=query)
devices = []
for dev_brain in search_results.results:
devices.append(dev_brain.getObject())
return devices
def get_location(location):
cat = IModelCatalogTool(dmd.Locations)
query = {}
query["meta_type"] = "Location"
query["uid"] = "/zport/dmd/Locations/*{0}".format(location)
res = cat.search(query=query)
location = None
if res.total > 0:
locationUid = str(next(res.results).uid)
location = dmd.unrestrictedTraverse(locationUid)
return location
def getInstances(self, uid=None, start=0, limit=50, sort='name',
dir='ASC', params=None):
# do the catalog search
cat = IModelCatalogTool(self._getObject(uid))
reverse = bool(dir == 'DESC')
brains = cat.search(self._instanceClass, start=start, limit=limit,
orderby=sort, reverse=reverse)
objs = imap(unbrain, brains)
# convert to info objects
return SearchResults(imap(IInfo, objs), brains.total, brains.hash_)
def _processSearch(self, limit=None, start=None, sort='name', dir='ASC',
params=None, uid=None, criteria=()):
ob = self._getObject(uid) if isinstance(uid, basestring) else uid
cat = IModelCatalogTool(ob)
query = {}
if params and params.get('name'):
query['name'] = "*{0}*".format(params.get('name'))
reverse = dir=='DESC'
return cat.search("Products.ZenModel.OSProcessClass.OSProcessClass",
start=start, limit=limit, orderby=sort,
reverse=reverse, query=query)
def _get_component_brains_from_model_catalog(self, uid, meta_type=()):
""" """
model_catalog = IModelCatalogTool(self.context.dmd)
query = {}
if meta_type:
query["meta_type"] = meta_type
query["objectImplements"] = "Products.ZenModel.DeviceComponent.DeviceComponent"
query["deviceId"] = uid
model_query_results = model_catalog.search(query=query)
brains = [ brain for brain in model_query_results.results ]
return brains
def getSubComponents(self):
cat = IModelCatalogTool(self)
# @TODO: Can we avoid NOTs ?
query = Eq('objectImplements', 'Products.ZenModel.DeviceComponent.DeviceComponent')
brains = cat.search(query=query)
children = []
for brain in brains:
try:
children.append(brain.getObject())
except:
pass
return children
def ip_assigned(ip):
cat = IModelCatalogTool(dmd.Networks)
query = {}
query["meta_type"] = "IpAddress"
query["uid"] = "*{0}".format(ip)
query["deviceId"] = "*"
res = cat.search(query=query)
device = None
if res.total > 0:
deviceId = str(next(res.results).deviceId)
device = dmd.unrestrictedTraverse(deviceId)
return device
def remote_createAllUsers(self):
cat = IModelCatalogTool(self.dmd)
brains = cat.search(("Products.ZenModel.Device.Device", "Products.ZenModel.DeviceClass.DeviceClass"))
users = []
for brain in brains:
device = brain.getObject()
user = self._create_user(device)
if user is not None:
users.append(user)
fmt = 'SnmpTrapConfig.remote_createAllUsers {0} users'
log.debug(fmt.format(len(users)))
return users
def _getSearchResultsFromModelCatalog(self,
parsedQuery,
sorter=None,
category=None,
countOnly=False,
unrestricted=False,
filterFn=None,
maxResults=None):
operators = parsedQuery.operators
keywords = parsedQuery.keywords
if not keywords:
return
def listMatchGlob(op, index, list):
return op(*[MatchGlob(index, '*%s*' % i) for i in list])
dmd = self._dmd
kw_query = Or(listMatchGlob(And, 'name', keywords),
listMatchGlob(And, 'text_ipAddress', keywords))
# Rank devices whose name match the query higher than other stuff
# TODO: Figure out how to expose Lucene boosts
# For now we just or the boost query in with the original query to boost those results
ranker = listMatchGlob(Or, 'name', keywords)
full_query = Or(kw_query, ranker)
cat = IModelCatalogTool(dmd).devices
limit = 0 if countOnly and not filterFn else maxResults
# Set orderby to None so that modelindex will rank by score
catalogItems = cat.search(query=full_query,
orderby=None,
filterPermissions=True,
limit=limit)
brainResults = [
DeviceSearchResult(catalogItem) for catalogItem in catalogItems
]
if countOnly and not filterFn:
return dict(Device=brainResults.total)
if filterFn:
brainResults = filter(filterFn, brainResults)
if countOnly:
return dict(Device=len(brainResults))
results = brainResults
if sorter is not None:
results = sorter.limitSort(results)
return results
def getSubComponents(self):
cat = IModelCatalogTool(self)
# @TODO: Can we avoid NOTs ?
query = Eq('objectImplements',
'Products.ZenModel.DeviceComponent.DeviceComponent')
brains = cat.search(query=query)
children = []
for brain in brains:
try:
children.append(brain.getObject())
except:
pass
return children
def getObjectBrains(self,
uid=None,
start=0,
limit=50,
sort='name',
dir='ASC',
params=None,
hashcheck=None,
types=(),
fields=[]):
cat = IModelCatalogTool(self._getObject(uid))
reverse = bool(dir == 'DESC')
qs = []
query = None
globFilters = {}
prodStates = None
params = params if params else {}
for key, value in params.iteritems():
if key == 'ipAddress':
qs.append(MatchGlob('text_ipAddress', '{}*'.format(value)))
elif key == 'productionState':
qs.append(
Or(*[Eq('productionState', str(state))
for state in value]))
# ZEN-30949 - stringify values from the 'priority' list if it's passed in for query criteria
elif key == 'priority':
qs.append(
Or(*[Eq('priority', str(priority)) for priority in value]))
# ZEN-10057 - move filtering on indexed groups/systems/location from post-filter to query
elif key in organizersToClass:
organizerQuery = self.findMatchingOrganizers(
organizersToClass[key], organizersToPath[key], value)
if not organizerQuery:
return []
qs.append(organizerQuery)
else:
globFilters[key] = value
if qs:
query = And(*qs)
return cat.search(types,
start=start,
limit=limit,
orderby=sort,
reverse=reverse,
query=query,
globFilters=globFilters,
hashcheck=hashcheck,
fields=fields)
def _get_component_types_from_model_catalog(self, uid):
""" """
componentTypes = {}
uuidMap = {}
model_catalog = IModelCatalogTool(self.context.dmd)
model_query = Eq('objectImplements', "Products.ZenModel.DeviceComponent.DeviceComponent")
model_query = And(model_query, Eq("deviceId", uid))
model_query_results = model_catalog.search(query=model_query, fields=["uuid", "meta_type"])
for brain in model_query_results.results:
uuidMap[brain.uuid] = brain.meta_type
compType = componentTypes.setdefault(brain.meta_type, { 'count' : 0, 'severity' : 0 })
compType['count'] += 1
return (componentTypes, uuidMap)
def validate_templates(self):
""" Check that both catalogs return same data for templates """
global_catalog = ICatalogTool(dmd.Devices)
model_catalog = IModelCatalogTool(dmd.Devices)
# get template nodes from global catalog
global_catalog_brains = global_catalog.search(types=('Products.ZenModel.RRDTemplate.RRDTemplate',))
global_catalog_templates = set([ brain.getPath() for brain in global_catalog_brains ])
# get template nodes from model catalog
model_catalog_brains = global_catalog.search(types=('Products.ZenModel.RRDTemplate.RRDTemplate',))
model_catalog_templates = set([ brain.getPath() for brain in model_catalog_brains ])
# compare results
if len(model_catalog_templates - global_catalog_templates) == 0 and \
len(global_catalog_templates - model_catalog_templates) == 0:
for template in global_catalog_templates:
template_object = dmd.unrestrictedTraverse(template)
query = Eq('id', template_object.id)
gc_brains = global_catalog.search(types=('Products.ZenModel.RRDTemplate.RRDTemplate',), query=query)
gc_templates = set([ brain.getPath() for brain in gc_brains ])
mc_brains = model_catalog.search(types=('Products.ZenModel.RRDTemplate.RRDTemplate',), query=query)
mc_templates = set([ brain.getPath() for brain in mc_brains ])
failed_templates = []
if not (len(mc_templates - gc_templates) == 0 and \
len(gc_templates - mc_templates) == 0):
failed_templates.append(template)
if failed_templates:
print "TEST FAILED: Inconsistent results from catalogs for templates:"
for failure in failed_templates:
print "\t{0}".format(failure)
else:
print "TEST PASSED: Both catalogs returned same results!!"
return True
else:
print "TEST FAILED: Inconsistent results from catalogs:"
print "\t{0}".format("Templates found in global catalog and not in model catalog: {0}".format(global_catalog_templates - model_catalog_templates))
print "\t{0}".format("Templates found in model catalog and not in global catalog: {0}".format(model_catalog_templates - global_catalog_templates))
return False
def processOrganizerUpdated(self, object, event):
catalog = IModelCatalogTool(object.primaryAq())
results = catalog.search(OSProcessClass)
if not results.total:
return
devices = set()
for organizer in results:
if results.areBrains:
organizer = organizer.getObject()
for process in organizer.instances():
device = process.device()
if not device:
continue
device = device.primaryAq()
device_path = device.getPrimaryUrlPath()
if not device_path in devices:
self._notifyAll(device)
devices.add(device_path)
def getCopyTargets(self, uid, query=''):
catalog = IModelCatalogTool(self._dmd)
template = self._getObject(uid)
types = ['Products.ZenModel.DeviceClass.DeviceClass']
brains = catalog.search(types=types)
objs = imap(unbrain, brains)
def genTargets():
for obj in objs:
container = obj.rrdTemplates
organizer = '/' + '/'.join(obj.getPrimaryPath()[4:])
label = organizer
if template.id in container.objectIds():
label += " (%s)" % _t('Create Copy')
if label.lower().startswith(query.lower()):
uid = '/'.join(obj.getPrimaryPath())
yield dict(uid=uid, label=label)
def byLabel(left, right):
return cmp(left['label'].lower(), right['label'].lower())
return sorted(genTargets(), byLabel)
def __call__(self, query='', dataRoot='devices'):
"""
@param query: A glob by which to filter device names
@type query: str
@return: A JSON representation of a list of ids
@rtype: "['id1', 'id2', 'id3']"
"""
if dataRoot != 'devices':
import exceptions
raise exceptions.ValueError("dataRoot should only be 'devices'")
query_scope = self.context.dmd.Devices
query = MatchGlob('name', query.rstrip('*') + '*')
if isinstance(self.context, DeviceOrganizer):
query_scope = self.context
catalog = IModelCatalogTool(query_scope).devices
brains = catalog.search(query=query, fields=['name'])
return sorted((b.name for b in brains), key=lambda x: x.lower())
def _get_brains(self, layer, attr, value):
"""
hack to make getLinkedNodes's awful code work with as little changes as possible
"""
model_catalog = IModelCatalogTool(self.dmd)
query = {}
if layer == 3:
model_catalog = model_catalog.layer3
meta_type = "IpAddress"
query["deviceId"] = "*" # We only are interested in assigned ips
else:
model_catalog = model_catalog.layer2
meta_type = "IpInterface"
query["meta_type"] = meta_type
if isinstance(value, basestring):
value = "*{0}".format(value)
query[attr] = value
search_results = model_catalog.search(query=query)
return [ brain for brain in search_results.results ]
def getDevProdStateJSON(self, prodStates=['Maintenance']):
"""
Return a map of device to production state in a format suitable for a
YUI data table.
@return: A JSON representation of a dictionary describing devices
@rtype: "{
'columns':['Device', 'Prod State'],
'data':[
{'Device':'<a href=/>', 'Prod State':'Production'},
{'Device':'<a href=/>', 'Prod State':'Maintenance'},
]}"
"""
if isinstance(prodStates, basestring):
prodStates = [prodStates]
def getProdStateInt(prodStateString):
for t in self.context.getProdStateConversions():
if t[0] == prodStateString:
return t[1]
numericProdStates = [getProdStateInt(p) for p in prodStates]
catalog = IModelCatalogTool(self.context.getPhysicalRoot().zport.dmd)
query = In('productionState', numericProdStates)
query = And(query,
Eq('objectImplements', 'Products.ZenModel.Device.Device'))
objects = list(catalog.search(query=query, orderby='id',
fields="uuid"))
devs = (x.getObject() for x in objects)
mydict = {'columns': ['Device', 'Prod State'], 'data': []}
for dev in devs:
if not self.context.checkRemotePerm(ZEN_VIEW, dev): continue
mydict['data'].append({
'Device': dev.getPrettyLink(),
'Prod State': dev.getProdState()
})
if len(mydict['data']) >= 100:
break
return mydict
def getInstances(self,
uid=None,
start=0,
limit=50,
sort='name',
dir='ASC',
params=None):
# do the catalog search
cat = IModelCatalogTool(self._getObject(uid))
reverse = bool(dir == 'DESC')
brains = cat.search(self._instanceClass,
start=start,
limit=limit,
orderby=sort,
reverse=reverse)
objs = imap(unbrain, brains)
# convert to info objects
return SearchResults(imap(IInfo, objs), brains.total, brains.hash_)
def _get_brains(self, layer, attr, value):
"""
hack to make getLinkedNodes's awful code work with as little changes as possible
"""
model_catalog = IModelCatalogTool(self.dmd)
query = {}
if layer == 3:
model_catalog = model_catalog.layer3
meta_type = "IpAddress"
query["deviceId"] = "*" # We only are interested in assigned ips
else:
model_catalog = model_catalog.layer2
meta_type = "IpInterface"
query["meta_type"] = meta_type
if isinstance(value, basestring):
value = "*{0}".format(value)
query[attr] = value
search_results = model_catalog.search(query=query)
return [brain for brain in search_results.results]
def _processSearch(self,
limit=None,
start=None,
sort='name',
dir='ASC',
params=None,
uid=None,
criteria=()):
ob = self._getObject(uid) if isinstance(uid, basestring) else uid
cat = IModelCatalogTool(ob)
query = {}
if params and params.get('name'):
query['name'] = "*{0}*".format(params.get('name'))
reverse = dir == 'DESC'
return cat.search("Products.ZenModel.OSProcessClass.OSProcessClass",
start=start,
limit=limit,
orderby=sort,
reverse=reverse,
query=query)
def __call__(self, query='', dataRoot='devices'):
"""
@param query: A glob by which to filter device names
@type query: str
@return: A JSON representation of a list of ids
@rtype: "['id1', 'id2', 'id3']"
"""
if dataRoot != 'devices':
import exceptions
raise exceptions.ValueError("dataRoot should only be 'devices'")
query_scope = self.context.dmd.Devices
query = MatchGlob('name', query.rstrip('*') + '*')
if isinstance(self.context, DeviceOrganizer):
query_scope = self.context
catalog = IModelCatalogTool(query_scope).devices
brains = catalog.search(query=query, fields=['name'])
return sorted((b.name for b in brains),
key=lambda x: x.lower())
def _getSearchResultsFromModelCatalog(self, parsedQuery, sorter=None, category=None, countOnly=False,
unrestricted=False, filterFn=None, maxResults=None):
operators = parsedQuery.operators
keywords = parsedQuery.keywords
if not keywords:
return
def listMatchGlob(op, index, list):
return op(*[MatchGlob(index, '*%s*' % i) for i in list])
dmd = self._dmd
kw_query = Or(listMatchGlob(And, 'name', keywords),
listMatchGlob(And, 'text_ipAddress', keywords))
# Rank devices whose name match the query higher than other stuff
# TODO: Figure out how to expose Lucene boosts
# For now we just or the boost query in with the original query to boost those results
ranker = listMatchGlob(Or, 'name', keywords)
full_query = Or(kw_query, ranker)
cat = IModelCatalogTool(dmd).devices
limit = 0 if countOnly and not filterFn else maxResults
# Set orderby to None so that modelindex will rank by score
catalogItems = cat.search(query=full_query, orderby=None, filterPermissions=True, limit=limit)
brainResults = [DeviceSearchResult(catalogItem) for catalogItem in catalogItems]
if countOnly and not filterFn:
return dict(Device=brainResults.total)
if filterFn:
brainResults = filter(filterFn, brainResults)
if countOnly:
return dict(Device=len(brainResults))
results = brainResults
if sorter is not None:
results = sorter.limitSort(results)
return results
def getDeviceByIpAddress(self, deviceName, collector="localhost", ipAddress=""):
# convert device name to an ip address
if not ipAddress:
if isip(deviceName):
ipAddress = deviceName
else:
try:
ipAddress = getHostByName(deviceName)
except socket.error:
# look for duplicate name
return self.context.Devices.findDeviceByIdExact(deviceName)
# find a device with the same ip on the same collector
cat = IModelCatalogTool(self.context.Devices)
query = And(Eq('text_ipAddress', ipAddress),
Eq('objectImplements', 'Products.ZenModel.Device.Device'))
search_results = cat.search(query=query)
for brain in search_results.results:
if brain.getObject().getPerformanceServerName() == collector:
return brain.getObject()
def getSubComponents(self, meta_type="", monitored=True):
"""
Return generator of components, by meta_type if specified
"""
catalog = IModelCatalogTool(self)
COMPONENT = 'Products.ZenModel.DeviceComponent.DeviceComponent'
monitorq, typeq = None, None
if monitored:
monitorq = Eq('monitored', '1')
if meta_type:
typeq = Eq('meta_type', meta_type)
queries = filter(None, (monitorq, typeq))
if queries:
query = And(*queries) if len(queries) > 1 else queries[0]
else:
query = None
for brain in catalog.search(COMPONENT, query=query):
try:
yield brain.getObject()
except KeyError:
log.warn("bad path '%s' in global catalog", brain.getPath())
def get_net_from_catalog(self, ip):
"""
Search in the network tree the IpNetwork ip belongs to.
return None if the network is not found
"""
net = None
cat = IModelCatalogTool(self.getNetworkRoot())
decimal_ip = ipToDecimal(ip)
query = {}
query["firstDecimalIp"] = "[ * TO {0} ]".format(decimal_ip)
query["lastDecimalIp"] = "[ {0} TO * ]".format(decimal_ip)
query["objectImplements"] = "Products.ZenModel.IpNetwork.IpNetwork"
fields = [ "firstDecimalIp", "lastDecimalIp" ]
result = cat.search(query=query, fields=fields)
if result.total > 0:
# networks found. if more than network is found, return the one
# whose lastDecimalIp - firstDecimalIp is the smallest
net_brains_tuples = [ ( net_brain, long(net_brain.lastDecimalIp) - long(net_brain.firstDecimalIp) ) for net_brain in result.results ]
net_brain_tuple = min(net_brains_tuples, key=lambda x: x[1])
net = net_brain_tuple[0].getObject()
return net
def validate_device_components(self):
""" search for devices components with both old a new catalogs """
model_catalog = IModelCatalogTool(dmd)
failed_devices = []
object_implements_query = Eq('objectImplements', "Products.ZenModel.DeviceComponent.DeviceComponent")
for device in self._get_all_devices():
device_catalog = device.componentSearch
device_catalog_components = defaultdict(list)
for brain in device_catalog():
device_catalog_components[brain.meta_type].append(brain.getPath())
device_uid = "/".join(device.getPrimaryPath())
model_query = And(object_implements_query, Eq("deviceId", device_uid))
model_query_results = model_catalog.search(query=model_query)
model_catalog_components = defaultdict(list)
for brain in model_query_results.results:
model_catalog_components[brain.meta_type].append(brain.getPath())
same_keys = len(device_catalog_components.keys()) == len(model_catalog_components.keys()) and \
len(set(device_catalog_components.keys()) - set(model_catalog_components.keys())) == 0
if not same_keys:
failed_devices.append(device)
else:
for meta_type in model_catalog_components.keys():
device_catalog_values = device_catalog_components[meta_type]
model_catalog_values = model_catalog_components[meta_type]
same_values = len(device_catalog_values) == len(model_catalog_values) and \
len(set(device_catalog_values) - set(model_catalog_values)) == 0
if not same_values:
failed_devices.append(device)
break
if failed_devices:
print "TEST FAILED: Catalogs return different components for the following devices:"
for dev in failed_devices:
print "\t\t{0}".format(dev.getPrimaryId())
else:
print "TEST PASSED: Both catalogs returned the same components for every device."
return len(failed_devices) == 0
def getIpAddresses(self,
limit=0,
start=0,
sort='ipAddressAsInt',
dir='DESC',
params=None,
uid=None,
criteria=()):
infos = []
cat = IModelCatalogTool(self._getObject(uid))
reverse = dir == 'DESC'
brains = cat.search("Products.ZenModel.IpAddress.IpAddress",
start=start,
limit=limit,
orderby=sort,
reverse=reverse)
for brain in brains:
infos.append(IInfo(unbrain(brain)))
devuuids = set(info.device.uuid for info in infos if info.device)
# get ping severities
zep = getFacade('zep')
pingSeverities = zep.getEventSeverities(devuuids,
severities=(),
status=(),
eventClass=Status_Ping)
self._assignPingStatuses(infos, pingSeverities)
# get snmp severities
snmpSeverities = zep.getEventSeverities(devuuids,
severities=(),
status=(),
eventClass=Status_Snmp)
self._assignSnmpStatuses(infos, snmpSeverities)
return SearchResults(infos, brains.total, brains.hash_)
def _getDeviceBatch(self, selectstatus='none', goodevids=[],
badevids=[], offset=0, count=50, filter='',
orderby='titleOrId', orderdir='asc'):
unused(count, offset, orderby, orderdir)
if not isinstance(goodevids, (list, tuple)):
goodevids = [goodevids]
if not isinstance(badevids, (list, tuple)):
badevids = [badevids]
if selectstatus == 'all':
idquery = ~In('id', badevids)
else:
idquery = In('id', goodevids)
devfilter = '(?is).*%s.*' % filter
filterquery = Or(
MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter)
)
query = Eq('uid', self.context.absolute_url_path()) & idquery
query = query & filterquery
catalog = IModelCatalogTool(self.context)
objects = catalog.search(query=query)
return [x['id'] for x in objects]
def getObjectBrains(self, uid=None, start=0, limit=50, sort='name',
dir='ASC', params=None, hashcheck=None, types=(), fields=[]):
cat = IModelCatalogTool(self._getObject(uid))
reverse = bool(dir == 'DESC')
qs = []
query = None
globFilters = {}
prodStates = None
params = params if params else {}
for key, value in params.iteritems():
if key == 'ipAddress':
qs.append(MatchGlob('text_ipAddress', '{}*'.format(value)))
elif key == 'productionState':
qs.append(Or(*[Eq('productionState', str(state))
for state in value]))
# ZEN-30949 - stringify values from the 'priority' list if it's passed in for query criteria
elif key == 'priority':
qs.append(Or(*[Eq('priority', str(priority))
for priority in value]))
# ZEN-10057 - move filtering on indexed groups/systems/location from post-filter to query
elif key in organizersToClass:
organizerQuery = self.findMatchingOrganizers(organizersToClass[key], organizersToPath[key], value)
if not organizerQuery:
return []
qs.append(organizerQuery)
else:
globFilters[key] = value
if qs:
query = And(*qs)
return cat.search(
types, start=start,
limit=limit, orderby=sort, reverse=reverse,
query=query, globFilters=globFilters, hashcheck=hashcheck, fields=fields)
class ModelCatalogTreeBuilder(object):
"""
Builds a Navigation Tree using Model Catalog
"""
def __init__(self, root, node_type, leaf_type, load_leaves=False, facet_field=None):
"""
@param root: root node of the tree
@param node_type: value of the object_implements field to query the catalog for nodes
Ex: "Products.ZenModel.DeviceOrganizer.DeviceOrganizer"
@param leaf_type: value of the object_implements field to query the catalog for leaves
Ex: "Products.ZenModel.Device.Device"
@param load_leaves: load leaves' brains from catalog or not
@param facet_field: field to retrieve leaf count using facets
"""
self.root = root # root object for which we are building the tree
self.brains = {} # obj_uid : obj brain
self.trees = {} # obj_uid : ModelCatalogTreeNode
self.root_tree = None
self.root_path = "/".join(self.root.getPrimaryPath())
self.model_catalog = IModelCatalogTool(self.root.dmd)
self.node_objectImplements = node_type
self.leaf_objectImplements = leaf_type
self.load_leaves = load_leaves
self.facet_field = facet_field
# Leaf brain field to use to determine the leaf's parent in case
# parent and leaves primary paths dont belong to the same tree.
# Example:
# Group: uid => /zport/dmd/Groups/my_group
# Device: uid => /zport/dmd/Devices/blabla/my_device
# we need a different field to get the parent - child relationship
#
self.parenthood_field = "uid"
self.get_node_uid_from_parenthood_field = lambda x: "/".join(x.split("/")[:-2])
if self.root_path.startswith("/zport/dmd/Groups") or \
self.root_path.startswith("/zport/dmd/Systems") or \
self.root_path.startswith("/zport/dmd/Locations"):
self.parenthood_field = "deviceOrganizers"
self.get_node_uid_from_parenthood_field = lambda x: x
# Build the tree
start = time.time()
self.build_tree()
msg = "Building tree for {} took {} seconds."
log.debug(msg.format(self.root_path, time.time()-start))
def _query_catalog(self, objectImplements, filter_permissions=True, limit=None, fields=None, facet_field=None):
requested_fields = set([ UID, "name", "id", "uuid", "meta_type" ])
params = {}
params["types"] = objectImplements
params["paths"] = self.root_path
if fields:
if isinstance(fields, basestring):
fields = [fields]
requested_fields |= set(fields)
params["fields"] = list(requested_fields)
params["filterPermissions"] = filter_permissions
if limit:
params["limit"] = limit
if facet_field:
params["facets_for_field"] = facet_field
return self.model_catalog.search(**params)
def build_tree(self):
"""
Builds the tree for a given root node and its subtrees
"""
# Load the nodes
search_results = self._query_catalog(self.node_objectImplements, filter_permissions=False)
nodes = set()
for brain in search_results.results:
self.brains[brain.uid] = brain
nodes.add(brain.uid)
self.root_tree = ModelCatalogTreeNode(self.root_path)
self.trees[self.root_path] = self.root_tree
for path in nodes:
if not path.startswith(self.root_path) or \
path == self.root_path:
continue
current_path = self.root_path
current_tree = self.root_tree
# get subtrees
# Example:
# root: /zport/dmd/Devices
# path: /zport/dmd/Devices/Server/Linux
# subtrees: Server
# Server/Linux
#
subtrees = path.replace(self.root_path, "").strip("/").split("/")
for subtree in subtrees:
if not subtree:
continue
current_path = "{}/{}".format(current_path, subtree)
if subtree not in current_tree.child_trees:
new_tree = ModelCatalogTreeNode(current_path)
current_tree.child_trees[subtree] = new_tree
self.trees[current_path] = new_tree
current_tree = current_tree.child_trees[subtree]
if self.load_leaves or not self.facet_field:
# We need to load all leaves
self._load_leaves()
self._load_leaf_counts() # gets count counting unique leaves
else:
# Get all the counts in just one solr call
self._load_leaf_counts_using_facets()
def _load_leaves(self):
"""
Loads the leaves for each node
"""
search_results = self._query_catalog(self.leaf_objectImplements, fields=self.parenthood_field)
for leaf_brain in search_results:
nodes = set()
self.brains[leaf_brain.uid] = leaf_brain
parenthood_field_value = getattr(leaf_brain, self.parenthood_field, None)
if parenthood_field_value:
if isinstance(parenthood_field_value, basestring):
parenthood_field_value = [parenthood_field_value]
# get the nodes this leaf belongs to
for v in parenthood_field_value:
node_uid = self.get_node_uid_from_parenthood_field(v)
if node_uid.startswith(self.root_path):
nodes.add(node_uid)
# add the leaf uid to the organizers it belongs to
for node_uid in nodes:
if node_uid in self.trees:
log.debug("adding leaf {} to node {} ".format(leaf_brain.uid, node_uid))
self.trees[node_uid].leaves.add(leaf_brain.uid)
def _load_leaf_counts(self):
# Load leaves for all nodes from the bottom up and
# calculate the leaf count
nodes_uids = self.trees.keys()
nodes_uids.sort(reverse=True, key=lambda x: len(x.split("/")))
for node_uid in nodes_uids:
current_tree = self.trees[node_uid]
all_leaves = set(current_tree.leaves)
for child_tree in current_tree.child_trees.itervalues():
all_leaves = all_leaves | child_tree.all_leaves
current_tree.all_leaves = all_leaves
current_tree.partial_leave_count = len(current_tree.leaves)
current_tree.total_leaf_count = len(current_tree.all_leaves)
def _load_leaf_counts_using_facets(self):
# facets values are returned in lower case
path_translator = {}
for tree_path in self.trees.iterkeys():
lower_case_path = tree_path.lower()
path_translator[lower_case_path] = tree_path
# query to retrieve the counts using facets
search_results = self._query_catalog(self.leaf_objectImplements, limit=0, facet_field=self.facet_field)
# load the partial child counts using facets
if search_results.facets and search_results.facets.get(self.facet_field):
facets = search_results.facets[self.facet_field]
for lower_case_path, count in facets.get_values().iteritems():
real_path = path_translator.get(lower_case_path)
if real_path:
self.trees[real_path].partial_leaf_count = count
# Calculate the total leaf count adding up the partial counts
nodes_uids = self.trees.keys()
nodes_uids.sort(reverse=True, key=lambda x: len(x.split("/")))
for node_uid in nodes_uids:
current_tree = self.trees[node_uid]
leaf_count = 0
for child_tree in current_tree.child_trees.itervalues():
leaf_count += child_tree.total_leaf_count
current_tree.total_leaf_count = current_tree.partial_leaf_count + leaf_count
def get_children(self, node_path):
tree = self.trees[node_path]
subtrees = tree.child_trees.values()
subtrees.sort( key=lambda x: x.id )
brains = [ self.brains[subtree.path] for subtree in subtrees if self.brains.get(subtree.path)]
return brains
def get_leaf_count(self, node_path):
count = 0
node_tree = self.trees.get(node_path)
if node_tree:
count = node_tree.total_leaf_count
return count
def _get_sorted_brains(self, uids, order_by="name"):
brains = [ self.brains[uid] for uid in uids ]
if brains and order_by:
brains.sort(reverse=False, key=lambda x: getattr(x, order_by, x.id).lower())
return brains
def get_child_nodes_brains(self, node_path, order_by="name"):
brains = []
if self.trees.get(node_path):
node_tree = self.trees[node_path]
children_uids = [ tree.path for tree in node_tree.child_trees.values() ]
brains = self._get_sorted_brains(children_uids, order_by)
return brains
def get_node_leaves(self, node_path, include_subnodes=False, order_by="name"):
"""
return the leaves brains for the received node
"""
brains = []
if self.load_leaves:
node_tree = self.trees[node_path]
leaves = node_tree.all_leaves if include_subnodes else node_tree.leaves
brains = self._get_sorted_brains(leaves, order_by)
return brains
class ModelCatalogHelper(object):
def __init__(self):
self.model_catalog = IModelCatalogTool(dmd)
def get_device_classes(self, return_paths=True):
query = Eq("objectImplements", "Products.ZenModel.DeviceClass.DeviceClass")
search_response = self.model_catalog.search(query=query)
results = search_response.results
if return_paths:
results = [ brain.getPath() for brain in results ]
return results
def get_devices_path_for_device_class(self, dc_path, return_paths=True):
query = [ Eq("objectImplements", "Products.ZenModel.Device.Device") ]
query.append(MatchGlob("path", "{0}/devices/*".format(dc_path)))
search_response = self.model_catalog.search(query=And(*query))
results = search_response.results
if return_paths:
results = [ brain.getPath() for brain in results ]
return results
def get_device_components(self, device, return_paths=True):
""" """
if not isinstance(device, basestring):
device = "/".join(device.getPrimaryPath())
query = []
query.append(Eq("objectImplements", "Products.ZenModel.DeviceComponent.DeviceComponent"))
query.append(Eq("deviceId", device))
search_response = self.model_catalog.search(query=And(*query))
results = search_response.results
if return_paths:
results = [ brain.getPath() for brain in results ]
return results
def get_device_ipInterfaces(self, device, return_paths=True):
""" """
if not isinstance(device, basestring):
device = "/".join(device.getPrimaryPath())
query = []
query.append(Eq("objectImplements", "Products.ZenModel.IpInterface.IpInterface"))
query.append(Eq("meta_type", "IpInterface"))
query.append(Eq("deviceId", device))
search_response = self.model_catalog.search(query=And(*query))
results = search_response.results
if return_paths:
results = [ brain.getPath() for brain in results ]
return results
def get_networks(self, return_paths=True):
query = []
query.append(Eq("objectImplements", "Products.ZenModel.IpNetwork.IpNetwork"))
query.append( Or( MatchGlob("uid", "/zport/dmd/Networks/*"), MatchGlob("uid", "/zport/dmd/IPv6Networks/*") ) )
search_response = self.model_catalog.search(query=And(*query))
results = search_response.results
if return_paths:
results = [ brain.getPath() for brain in results ]
return results
def get_device_ipaddresses(self, device, return_paths=True):
if not isinstance(device, basestring):
device = "/".join(device.getPrimaryPath())
query = []
query.append(Eq("objectImplements", "Products.ZenModel.IpAddress.IpAddress"))
query.append(Eq("meta_type", "IpAddress"))
query.append(Eq("deviceId", device))
search_response = self.model_catalog.search(query=And(*query))
results = search_response.results
if return_paths:
results = [ brain.getPath() for brain in results ]
return results
def get_device_processes(self, device, return_paths=True):
if not isinstance(device, basestring):
device = "/".join(device.getPrimaryPath())
query = []
query.append(Eq("objectImplements", "Products.ZenModel.OSProcess.OSProcess"))
query.append(MatchGlob("path", "{0}*".format(device)))
search_response = self.model_catalog.search(query=And(*query))
results = search_response.results
if return_paths:
results = [ brain.getPath() for brain in results ]
return results
def get_doc_by_uid(self, uid):
if not isinstance(uid, basestring):
uid = "/".join(uid.getPrimaryPath())
query = 'uid:"{0}"'.format(uid)
search_response = self.model_catalog.search(query=query)
results = search_response.results
if return_paths:
results = [ brain.getPath() for brain in results ]
return results
def get_network_ips(self, net, return_paths=True):
if not isinstance(net, basestring):
net = "/".join(net.getPrimaryPath())
query = []
query.append(Eq("objectImplements", "Products.ZenModel.IpAddress.IpAddress"))
query.append(Eq("networkId", net))
search_response = self.model_catalog.search(query=And(*query))
results = search_response.results
if return_paths:
results = [ brain.getPath() for brain in results ]
return results
def get_object_paths(self, obj):
paths = []
if not isinstance(obj, basestring):
obj = "/".join(obj.getPrimaryPath())
search_response = self.model_catalog.search(query=Eq("uid", obj))
if search_response.total == 1:
brain = next(search_response.results)
if brain.path is not None:
paths = brain.path
else:
paths = [ "found more than 1 uid for {0}".format(obj) ]
return paths
class ZenossAppData(object):
implements(IZenossData)
def callHomeData(self, dmd):
self.dmd = dmd
self._catalog = IModelCatalogTool(self.dmd)
stats = (self.server_key,
self.google_key,
self.all_versions,
self.event_classes,
self.event_count,
self.reports,
self.templates,
self.systems,
self.groups,
self.locations,
self.total_collectors,
self.zenpacks,
self.user_count,
self.product_count,
self.product_name,
self.components)
return chain.from_iterable(map(lambda fn: fn(), stats))
def components(self):
brains = self._catalog.search(types=(DeviceComponent,), facets_for_field=["meta_type"])
if brains.facets and brains.facets.get("meta_type"):
facets = brains.facets["meta_type"]
comps = facets.get_values()
for comp, count in comps.iteritems():
if count > 0:
yield ("Components",
"{}: {}".format(comp, count))
yield ("Components:", [])
def product_name(self):
yield "Product", self.dmd.getProductName()
def product_count(self):
manufacturers = self.dmd.Manufacturers.objectValues(
spec='Manufacturer')
prodCount = 0
for m in manufacturers:
prodCount += m.products.countObjects()
yield "Product Count", prodCount
def user_count(self):
yield "User Count", len(self.dmd.ZenUsers.objectValues(spec="UserSettings"))
def server_key(self):
key = self.dmd.uuid or "NOT ACTIVATED"
yield "Server Key", key
def google_key(self):
yield "Google Key", self.dmd.geomapapikey
def zenpacks(self):
for zenpack in self.dmd.ZenPackManager.packs():
yield ("Zenpack",
"{zenpack.id} {zenpack.version}".format(**locals()))
def all_versions(self):
zenoss_version, cc_version = self.dmd.About.getZenossVersion(), self.dmd.About.getControlCenterVersion()
yield zenoss_version.name, zenoss_version.full()
yield cc_version.name, cc_version.full()
def event_classes(self):
yield 'Evt Mappings', self.dmd.Events.countInstances()
def reports(self):
yield "Reports", self.dmd.Reports.countReports()
def templates(self):
yield "Templates", len(self.dmd.searchRRDTemplates)
def systems(self):
yield "Systems", self.dmd.Systems.countChildren()
def groups(self):
yield "Groups", self.dmd.Groups.countChildren()
def locations(self):
yield "Locations", self.dmd.Locations.countChildren()
def total_collectors(self):
results = self.dmd.Monitors.getPerformanceMonitorNames()
yield "Collectors", len(results)
def event_count(self):
zep = getFacade('zep', self.dmd)
try:
yield ("Event Count",
zep.countEventsSince(time.time() - 24 * 60 * 60))
except ZepConnectionError:
yield "Event Count: last 24hr", "Not Available"
