def validate_devices_in_networks(self):
"""
Get deviceId's of every network using both catalogs and compare results
Network -> Devices
"""
networks = get_networks(dmd)
layer3_catalog = dmd.ZenLinkManager.layer3_catalog
model_catalog = IModelCatalogTool(dmd)
failed_networks = {}
for network in networks:
# Devices under device class in global catalog
query = Eq('networkId', network)
layer3_brains = layer3_catalog.evalAdvancedQuery(query)
layer3_device_ids = set([ brain.deviceId for brain in layer3_brains if brain.deviceId ])
model_catalog_brains = model_catalog.search(query=query)
model_catalog_device_ids = set([ brain.deviceId.split("/")[-1] for brain in model_catalog_brains.results if brain.deviceId ])
if not len(layer3_device_ids - model_catalog_device_ids) == len(model_catalog_device_ids - layer3_device_ids) == 0:
#import pdb; pdb.set_trace()
failed_networks[network] = (layer3_device_ids, model_catalog_device_ids)
if failed_networks:
print "TEST FAILED: Catalogs return different devices for the following networks:"
print "\t\t{0}".format(failed_networks.keys())
else:
print "TEST PASSED: Both catalogs returned the same devices for all networks."
return len(failed_networks) == 0
def _find_instances_in_catalog(self, count=False):
model_catalog = IModelCatalogTool(self.dmd)
query = { "productClassId": self.idx_uid() }
if count:
return model_catalog.search(query=query, limit=0)
else:
return model_catalog.search(query=query)
def moveProcess(self, uid, targetUid):
obj = self._getObject(uid)
target = self._getObject(targetUid)
brainsCollection = []
# reindex all the devices and processes underneath this guy and the target
for org in (obj.getPrimaryParent().getPrimaryParent(), target):
catalog = IModelCatalogTool(org)
brainsCollection.append(catalog.search(OSProcess))
if isinstance(obj, OSProcessClass):
source = obj.osProcessOrganizer()
source.moveOSProcessClasses(targetUid, obj.id)
newObj = getattr(target.osProcessClasses, obj.id)
elif isinstance(obj, OSProcessOrganizer):
source = aq_parent(obj)
source.moveOrganizer(targetUid, (obj.id, ))
newObj = getattr(target, obj.id)
else:
raise Exception('Illegal type %s' % obj.__class__.__name__)
# fire the object moved event for the process instances (will update catalog)
for brains in brainsCollection:
objs = imap(unbrain, brains)
for item in objs:
notify(
ObjectMovedEvent(item, item.os(), item.id, item.os(),
item.id))
return newObj.getPrimaryPath()
def _search_catalog(self, obj, types=(), paths=(), query=None):
if USE_MODEL_CATALOG:
catalog = IModelCatalogTool(obj)
else:
catalog = ICatalogTool(obj)
return catalog.search(types=types, paths=paths, query=query)
def moveProcess(self, uid, targetUid):
obj = self._getObject(uid)
target = self._getObject(targetUid)
brainsCollection = []
# reindex all the devices and processes underneath this guy and the target
for org in (obj.getPrimaryParent().getPrimaryParent(), target):
catalog = IModelCatalogTool(org)
brainsCollection.append(catalog.search(OSProcess))
if isinstance(obj, OSProcessClass):
source = obj.osProcessOrganizer()
source.moveOSProcessClasses(targetUid, obj.id)
newObj = getattr(target.osProcessClasses, obj.id)
elif isinstance(obj, OSProcessOrganizer):
source = aq_parent(obj)
source.moveOrganizer(targetUid, (obj.id,))
newObj = getattr(target, obj.id)
else:
raise Exception('Illegal type %s' % obj.__class__.__name__)
# fire the object moved event for the process instances (will update catalog)
for brains in brainsCollection:
objs = imap(unbrain, brains)
for item in objs:
notify(ObjectMovedEvent(item, item.os(), item.id, item.os(), item.id))
return newObj.getPrimaryPath()
def testDeleteOrganizerRemovesDevices(self):
"""When we delete an organizer all the devices assigned to it should not
still have a relationship to it in the catalog
"""
# Create Organizer
organizer = self.facade.addOrganizer("/zport/dmd/Groups",
'testOrganizer')
organizer_path = organizer.uid
catalog = IModelCatalogTool(self.dmd)
# Add some devices to it (use createInstance to create a device)
devices = self.dmd.Devices
test_device = devices.createInstance('testDevice')
self.facade.moveDevices(['/'.join(test_device.getPhysicalPath())],
organizer_path)
organizer = self.dmd.unrestrictedTraverse(organizer_path)
# test we have added the device
self.assertEqual(len(organizer.getDevices()), 1,
"make sure we saved our device")
deviceBrains = catalog.search(
paths='/'.join(organizer.getPhysicalPath()))
self.assertTrue(
deviceBrains.total > 1,
" At this point we should have the organizer and the device")
# Delete the Organizer
self.facade.deleteNode(organizer_path)
# Get the devices directly from the path
deviceBrains = catalog.search(
paths='/'.join(organizer.getPhysicalPath()))
self.assertEqual(deviceBrains.total, 0,
" we should not have any devices at this point")
def _find_instances_in_catalog(self, count=False):
model_catalog = IModelCatalogTool(self.dmd)
query = {"productClassId": self.idx_uid()}
if count:
return model_catalog.search(query=query, limit=0)
else:
return model_catalog.search(query=query)
def _getDeviceBatch(self,
selectstatus='none',
goodevids=[],
badevids=[],
offset=0,
count=50,
filter='',
orderby='titleOrId',
orderdir='asc'):
unused(count, offset, orderby, orderdir)
if not isinstance(goodevids, (list, tuple)):
goodevids = [goodevids]
if not isinstance(badevids, (list, tuple)):
badevids = [badevids]
if selectstatus == 'all':
idquery = ~In('id', badevids)
else:
idquery = In('id', goodevids)
devfilter = '(?is).*%s.*' % filter
filterquery = Or(MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter))
query = Eq('uid', self.context.absolute_url_path()) & idquery
query = query & filterquery
catalog = IModelCatalogTool(self.context)
objects = catalog.search(query=query)
return [x['id'] for x in objects]
def _getAdvancedQueryDeviceList(self, offset=0, count=50, filter='',
orderby='name', orderdir='asc'):
"""
Ask the catalog for devices matching the criteria specified.
"""
context = self.context
if not isinstance(context, DeviceOrganizer):
context = self.context.dmd.Devices
catalog = IModelCatalogTool(context).devices
devfilter = '(?is).*%s.*' % filter
filterquery = Or(
MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter)
)
query = Eq('uid', context.absolute_url_path()
) & filterquery
objects = catalog.search(
query=query,
start=int(offset),
limit=int(count),
order_by=orderby,
reverse=orderdir != 'asc'
)
objects = list(objects)
totalCount = len(objects)
return totalCount, objects
def validate_devices_interfaces_and_mac_addresses(self):
"""
get macs of avery device in the system with both catalogs
"""
layer2_catalog = dmd.ZenLinkManager.layer2_catalog
model_catalog = IModelCatalogTool(dmd)
failed_devices = []
for device in get_all_devices(dmd):
device_uid = "/".join(device.getPrimaryPath())
layer2_query = Eq('deviceId', device_uid)
layer2_brains = layer2_catalog.evalAdvancedQuery(layer2_query)
layer2_macs = set([ brain.macaddress for brain in layer2_brains if brain.macaddress ])
layer2_ifaces = set([ brain.interfaceId for brain in layer2_brains if brain.interfaceId ])
model_catalog_query = And(Eq('deviceId', "{0}".format(device_uid)), Eq('meta_type', "IpInterface"))
search_results = model_catalog.search(query=model_catalog_query)
model_catalog_brains = [ brain for brain in search_results.results ]
model_catalog_macs = set([ brain.macaddress for brain in model_catalog_brains if brain.macaddress ])
model_catalog_ifaces = set([ brain.interfaceId for brain in model_catalog_brains if brain.interfaceId ])
if not ( len(layer2_macs - model_catalog_macs) == len(model_catalog_macs- layer2_macs) == 0 and \
len(layer2_ifaces - model_catalog_ifaces) == len(model_catalog_ifaces- layer2_ifaces) == 0 ):
failed_devices.append(device)
if failed_devices:
print "TEST FAILED: Catalogs return different mac addresses or interfaces for the following devices:"
for dev in failed_devices:
print "\t\t{0}".format(dev.getPrimaryId())
else:
print "TEST PASSED: Both catalogs returned the same mac addresses and interfaces for every device."
return len(failed_devices) == 0
def _getAdvancedQueryDeviceList(self,
offset=0,
count=50,
filter='',
orderby='name',
orderdir='asc'):
"""
Ask the catalog for devices matching the criteria specified.
"""
context = self.context
if not isinstance(context, DeviceOrganizer):
context = self.context.dmd.Devices
catalog = IModelCatalogTool(context).devices
devfilter = '(?is).*%s.*' % filter
filterquery = Or(MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter))
query = Eq('uid', context.absolute_url_path()) & filterquery
objects = catalog.search(query=query,
start=int(offset),
limit=int(count),
order_by=orderby,
reverse=orderdir != 'asc')
objects = list(objects)
totalCount = len(objects)
return totalCount, objects
def validate_ips_and_networks_for_device(self):
"""
for every device, get the ips and networks
Device -> Ip Addresses
Device -> Network
"""
layer3_catalog = dmd.ZenLinkManager.layer3_catalog
model_catalog = IModelCatalogTool(dmd)
failed_devices = []
for device in get_all_devices(dmd):
l3_query = Eq('deviceId', device.id)
layer3_brains = layer3_catalog.evalAdvancedQuery(l3_query)
layer3_ips = set([ brain.ipAddressId for brain in layer3_brains if brain.ipAddressId ])
layer3_nets = set([ brain.networkId for brain in layer3_brains if brain.networkId ])
model_catalog_query = And(Eq('deviceId', "*{0}".format(device.id)), Eq('meta_type', "IpAddress"))
search_results = model_catalog.search(query=model_catalog_query)
model_catalog_brains = [ brain for brain in search_results.results ]
model_catalog_ips = set([ brain.ipAddressId for brain in model_catalog_brains if brain.ipAddressId ])
model_catalog_nets = set([ brain.networkId for brain in model_catalog_brains if brain.networkId ])
if not (len(layer3_ips - model_catalog_ips) == len(model_catalog_ips- layer3_ips) == 0 and \
len(layer3_nets - model_catalog_nets) == len(model_catalog_nets- layer3_nets) == 0) :
failed_devices.append(device)
if failed_devices:
print "TEST FAILED: Catalogs return different ip addresses for the following devices:"
for dev in failed_devices:
print "\t\t{0}".format(dev.getPrimaryId())
else:
print "TEST PASSED: Both catalogs returned the same ips for every device."
return len(failed_devices) == 0
def test_device_classes_devices(self):
""" Check devices under device classes are the same """
failed_device_classes = []
for dc in DEVICE_CLASSES:
dc_object = dmd.unrestrictedTraverse(dc)
# Devices under device class in global catalog
global_catalog = ICatalogTool(dc_object)
global_catalog_brains = global_catalog.search('Products.ZenModel.Device.Device')
global_catalog_results = set([ brain.getPath() for brain in global_catalog_brains.results ])
# Devices under device class in model catalog
model_catalog = IModelCatalogTool(dc_object)
model_catalog_brains = model_catalog.search('Products.ZenModel.Device.Device', limit=10000)
model_catalog_results = set([ brain.getPath() for brain in model_catalog_brains.results ])
result = "FAILED"
if len(global_catalog_results - model_catalog_results) == 0 and \
len(model_catalog_results-global_catalog_results) ==0:
result = "PASSED"
else:
failed_device_classes.append(dc)
if not failed_device_classes:
print "TEST PASSED: All devices found in the same device classes for both catalogs!!"
else:
print "TEST FAILED: The following device classes have different devices in the catalogs:"
for failed in failed_device_classes:
print "\t{0}".format(failed)
return len(failed_device_classes) == 0
def getAddTemplateTargets(self, query=None):
"""
@returns list of targets for our new template
"""
cat = IModelCatalogTool(self._dmd)
results = []
# it can be any device class that we add the template too
brains = cat.search(types=[DeviceClass])
for brain in brains:
# HACK: use the path to get the organizer name so we don't have to wake up the object
label = brain.getPath()
if label == "/zport/dmd/Devices":
label = "/"
else:
label = label.replace("/zport/dmd/Devices/", "/")
results.append(dict(uid=brain.getPath(), label=label))
# the display is organizer name and we do not index it
# so filter after the query
if query is not None:
results = [
result for result in results
if query.lower() in result['label'].lower()
]
return sorted(results, key=lambda org: org['label'])
class CatalogTool(object):
implements(ICatalogTool)
def __init__(self, context):
self.context = context
self.model_catalog = IModelCatalogTool(self.context)
def getBrain(self, path):
return self.model_catalog.global_catalog.getBrain(path)
def parents(self, path):
return self.model_catalog.parents(path)
def count(self, types=(), path=None, filterPermissions=True):
return self.model_catalog.count(types=types, path=path, filterPermissions=filterPermissions)
def search(self, types=(), start=0, limit=None, orderby=None,
reverse=False, paths=(), depth=None, query=None,
hashcheck=None, filterPermissions=True, globFilters=None):
return self.model_catalog.global_catalog.search(types=types, start=start, limit=limit, orderby=orderby,
reverse=reverse, paths=paths, depth=depth, query=query,
hashcheck=hashcheck, filterPermissions=filterPermissions, globFilters=globFilters)
def update(self, obj):
self.model_catalog.update(obj)
def getIpAddresses(self, limit=0, start=0, sort='ipAddressAsInt', dir='DESC',
params=None, uid=None, criteria=()):
infos = []
cat = IModelCatalogTool(self._getObject(uid))
reverse = dir=='DESC'
brains = cat.search("Products.ZenModel.IpAddress.IpAddress",
start=start, limit=limit,
orderby=sort, reverse=reverse)
for brain in brains:
infos.append(IInfo(unbrain(brain)))
devuuids = set(info.device.uuid for info in infos if info.device)
# get ping severities
zep = getFacade('zep')
pingSeverities = zep.getEventSeverities(devuuids,
severities=(),
status=(),
eventClass=Status_Ping)
self._assignPingStatuses(infos, pingSeverities)
# get snmp severities
snmpSeverities = zep.getEventSeverities(devuuids,
severities=(),
status=(),
eventClass=Status_Snmp)
self._assignSnmpStatuses(infos, snmpSeverities)
return SearchResults(infos, brains.total, brains.hash_)
def cutover(self, dmd):
log.info("Updating relationships on Ip Addresses")
catalog = IModelCatalogTool(dmd.Networks)
for ip in catalog.search(IpAddress):
try:
ip.getObject().buildRelations()
#some objects can fail relations building process
except (Exception, ):
continue
log.info("Updating relationships on Devices")
for dev in dmd.Devices.getSubDevices():
try:
dev.buildRelations()
if dev.manageIp:
ipobj = dev.getNetworkRoot().findIp(dev.manageIp)
if ipobj:
dev.ipaddress.addRelation(ipobj)
else:
ipWithoutNetmask, netmask = ipAndMaskFromIpMask(
dev.manageIp)
ipobj = dev.getNetworkRoot().createIp(
ipWithoutNetmask, netmask)
dev.ipaddress.addRelation(ipobj)
notify(IndexingEvent(ipobj))
#some objects can fail relations building process
except (Exception, ):
continue
def testDeleteOrganizerRemovesDevices(self):
"""When we delete an organizer all the devices assigned to it should not
still have a relationship to it in the catalog
"""
# Create Organizer
organizer = self.facade.addOrganizer("/zport/dmd/Groups", 'testOrganizer')
organizer_path = organizer.uid
catalog = IModelCatalogTool(self.dmd)
# Add some devices to it (use createInstance to create a device)
devices = self.dmd.Devices
test_device = devices.createInstance('testDevice')
self.facade.moveDevices(['/'.join(test_device.getPhysicalPath())], organizer_path)
organizer = self.dmd.unrestrictedTraverse(organizer_path)
# test we have added the device
self.assertEqual(len(organizer.getDevices()), 1, "make sure we saved our device")
deviceBrains = catalog.search(paths='/'.join(organizer.getPhysicalPath()))
self.assertTrue(deviceBrains.total > 1, " At this point we should have the organizer and the device")
# Delete the Organizer
self.facade.deleteNode(organizer_path)
# Get the devices directly from the path
deviceBrains = catalog.search(paths='/'.join(organizer.getPhysicalPath()))
self.assertEqual(deviceBrains.total, 0, " we should not have any devices at this point")
class ModelCatalogScanInfo(CatalogScanInfo):
def __init__(self, dmd):
super(ModelCatalogScanInfo, self).__init__(dmd, 'model_catalog', '')
self.updates = []
if USE_MODEL_CATALOG:
self._catalog_tool = IModelCatalogTool(self.dmd)
def size(self):
search_results = self._catalog_tool.search(filterPermissions=False,
start=0,
limit=0)
return search_results.total
def get_brains(self):
return self._catalog_tool.cursor_search()
def uncatalog_object(self, uid):
self.updates.append(IndexUpdate(None, op=UNINDEX, uid=uid))
if len(self.updates) % 1000 == 0:
self.commit()
def exists(self):
return USE_MODEL_CATALOG
def commit(self):
if self.updates:
self._catalog_tool.model_index.process_batched_updates(
self.updates)
self.updates = []
def __init__(self, dmd, event_summary):
self._dmd = dmd
self._event_summary = event_summary
self._eventOccurrence = event_summary['occurrence'][0]
self._eventActor = self._eventOccurrence['actor']
self._eventDetails = self._findDetails(self._eventOccurrence)
self._catalog = IModelCatalogTool(dmd)
self._manager = IGUIDManager(dmd)
def afterSetUp(self):
super(TestModelCatalogTransactions, self).afterSetUp()
# Lets change the ModelCatalogTestDataManager with ModelCatalogDataManager
self.model_catalog = IModelCatalogTool(self.dmd)
self.data_manager = ModelCatalogDataManager('localhost:8983', self.dmd)
self.model_catalog.model_catalog_client._data_manager = self.data_manager
# get a reference to model_index to be able to fo checks bypassing the data manager
self.model_index = self.data_manager.model_index
def _search_ip_in_catalog(self, ip):
""" Return list of brains that match ip """
cat = IModelCatalogTool(self.getNetworkRoot())
query = {}
query["objectImplements"] = "Products.ZenModel.IpAddress.IpAddress"
query["id"] = ipwrap(ip)
search_result = cat.search(query=query)
return [brain for brain in search_result.results]
def test(self, dmd):
from Products.Zuul.catalog.interfaces import IModelCatalogTool
results = IModelCatalogTool(dmd.Devices).search(
'Products.ZenModel.DeviceOrganizer.DeviceOrganizer')
instances = IModelCatalogTool(
dmd.Devices).search('Products.ZenModel.Device.Device')
tree = PathIndexCache(results, instances, 'devices')
print tree
def _get_devices_under_path(self, path):
""" Returnd device's brains """
model_catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}".format(path)
fields = ["id", "path"]
result = model_catalog.search(query=query, fields=fields)
return result.results
def _get_devices_under_path(self, path):
""" Returnd device's brains """
model_catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}".format(path)
fields = ["id", "path"]
result = model_catalog.search(query=query, fields=fields)
return result.results
def _getSubdevices(self, brains=False):
""" Helper method to search for devices/devices brains under the current organizer """
catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}*".format("/".join(self.getPhysicalPath()))
if not brains:
return getObjectsFromModelCatalog(catalog, query, LOG)
else:
return catalog.search(query=query).results
def _getSubdevices(self, brains=False):
""" Helper method to search for devices/devices brains under the current organizer """
catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}*".format("/".join(self.getPhysicalPath()))
if not brains:
return getObjectsFromModelCatalog(catalog, query, LOG)
else:
return catalog.search(query=query).results
def _get_all_devices(self):
""" """
model_catalog = IModelCatalogTool(dmd)
query = {}
query['objectImplements'] = "Products.ZenModel.Device.Device"
search_results = model_catalog.search(query=query)
devices = []
for dev_brain in search_results.results:
devices.append(dev_brain.getObject())
return devices
def get_location(location):
cat = IModelCatalogTool(dmd.Locations)
query = {}
query["meta_type"] = "Location"
query["uid"] = "/zport/dmd/Locations/*{0}".format(location)
res = cat.search(query=query)
location = None
if res.total > 0:
locationUid = str(next(res.results).uid)
location = dmd.unrestrictedTraverse(locationUid)
return location
def _get_component_brains_from_model_catalog(self, uid, meta_type=()):
""" """
model_catalog = IModelCatalogTool(self.context.dmd)
query = {}
if meta_type:
query["meta_type"] = meta_type
query["objectImplements"] = "Products.ZenModel.DeviceComponent.DeviceComponent"
query["deviceId"] = uid
model_query_results = model_catalog.search(query=query)
brains = [ brain for brain in model_query_results.results ]
return brains
def getInstances(self, uid=None, start=0, limit=50, sort='name',
dir='ASC', params=None):
# do the catalog search
cat = IModelCatalogTool(self._getObject(uid))
reverse = bool(dir == 'DESC')
brains = cat.search(self._instanceClass, start=start, limit=limit,
orderby=sort, reverse=reverse)
objs = imap(unbrain, brains)
# convert to info objects
return SearchResults(imap(IInfo, objs), brains.total, brains.hash_)
def _processSearch(self, limit=None, start=None, sort='name', dir='ASC',
params=None, uid=None, criteria=()):
ob = self._getObject(uid) if isinstance(uid, basestring) else uid
cat = IModelCatalogTool(ob)
query = {}
if params and params.get('name'):
query['name'] = "*{0}*".format(params.get('name'))
reverse = dir=='DESC'
return cat.search("Products.ZenModel.OSProcessClass.OSProcessClass",
start=start, limit=limit, orderby=sort,
reverse=reverse, query=query)
def getSubComponents(self):
cat = IModelCatalogTool(self)
# @TODO: Can we avoid NOTs ?
query = Eq('objectImplements', 'Products.ZenModel.DeviceComponent.DeviceComponent')
brains = cat.search(query=query)
children = []
for brain in brains:
try:
children.append(brain.getObject())
except:
pass
return children
def remote_createAllUsers(self):
cat = IModelCatalogTool(self.dmd)
brains = cat.search(("Products.ZenModel.Device.Device", "Products.ZenModel.DeviceClass.DeviceClass"))
users = []
for brain in brains:
device = brain.getObject()
user = self._create_user(device)
if user is not None:
users.append(user)
fmt = 'SnmpTrapConfig.remote_createAllUsers {0} users'
log.debug(fmt.format(len(users)))
return users
def ip_assigned(ip):
cat = IModelCatalogTool(dmd.Networks)
query = {}
query["meta_type"] = "IpAddress"
query["uid"] = "*{0}".format(ip)
query["deviceId"] = "*"
res = cat.search(query=query)
device = None
if res.total > 0:
deviceId = str(next(res.results).deviceId)
device = dmd.unrestrictedTraverse(deviceId)
return device
def _getSearchResultsFromModelCatalog(self,
parsedQuery,
sorter=None,
category=None,
countOnly=False,
unrestricted=False,
filterFn=None,
maxResults=None):
operators = parsedQuery.operators
keywords = parsedQuery.keywords
if not keywords:
return
def listMatchGlob(op, index, list):
return op(*[MatchGlob(index, '*%s*' % i) for i in list])
dmd = self._dmd
kw_query = Or(listMatchGlob(And, 'name', keywords),
listMatchGlob(And, 'text_ipAddress', keywords))
# Rank devices whose name match the query higher than other stuff
# TODO: Figure out how to expose Lucene boosts
# For now we just or the boost query in with the original query to boost those results
ranker = listMatchGlob(Or, 'name', keywords)
full_query = Or(kw_query, ranker)
cat = IModelCatalogTool(dmd).devices
limit = 0 if countOnly and not filterFn else maxResults
# Set orderby to None so that modelindex will rank by score
catalogItems = cat.search(query=full_query,
orderby=None,
filterPermissions=True,
limit=limit)
brainResults = [
DeviceSearchResult(catalogItem) for catalogItem in catalogItems
]
if countOnly and not filterFn:
return dict(Device=brainResults.total)
if filterFn:
brainResults = filter(filterFn, brainResults)
if countOnly:
return dict(Device=len(brainResults))
results = brainResults
if sorter is not None:
results = sorter.limitSort(results)
return results
def getSubComponents(self):
cat = IModelCatalogTool(self)
# @TODO: Can we avoid NOTs ?
query = Eq('objectImplements',
'Products.ZenModel.DeviceComponent.DeviceComponent')
brains = cat.search(query=query)
children = []
for brain in brains:
try:
children.append(brain.getObject())
except:
pass
return children
def getObjectBrains(self,
uid=None,
start=0,
limit=50,
sort='name',
dir='ASC',
params=None,
hashcheck=None,
types=(),
fields=[]):
cat = IModelCatalogTool(self._getObject(uid))
reverse = bool(dir == 'DESC')
qs = []
query = None
globFilters = {}
prodStates = None
params = params if params else {}
for key, value in params.iteritems():
if key == 'ipAddress':
qs.append(MatchGlob('text_ipAddress', '{}*'.format(value)))
elif key == 'productionState':
qs.append(
Or(*[Eq('productionState', str(state))
for state in value]))
# ZEN-30949 - stringify values from the 'priority' list if it's passed in for query criteria
elif key == 'priority':
qs.append(
Or(*[Eq('priority', str(priority)) for priority in value]))
# ZEN-10057 - move filtering on indexed groups/systems/location from post-filter to query
elif key in organizersToClass:
organizerQuery = self.findMatchingOrganizers(
organizersToClass[key], organizersToPath[key], value)
if not organizerQuery:
return []
qs.append(organizerQuery)
else:
globFilters[key] = value
if qs:
query = And(*qs)
return cat.search(types,
start=start,
limit=limit,
orderby=sort,
reverse=reverse,
query=query,
globFilters=globFilters,
hashcheck=hashcheck,
fields=fields)
def _get_component_types_from_model_catalog(self, uid):
""" """
componentTypes = {}
uuidMap = {}
model_catalog = IModelCatalogTool(self.context.dmd)
model_query = Eq('objectImplements', "Products.ZenModel.DeviceComponent.DeviceComponent")
model_query = And(model_query, Eq("deviceId", uid))
model_query_results = model_catalog.search(query=model_query, fields=["uuid", "meta_type"])
for brain in model_query_results.results:
uuidMap[brain.uuid] = brain.meta_type
compType = componentTypes.setdefault(brain.meta_type, { 'count' : 0, 'severity' : 0 })
compType['count'] += 1
return (componentTypes, uuidMap)
def deleteNode(self, uid):
"""
Remove a report or report organizer.
@type uid: string
@param uid: The UID of the node to delete
@rtype: [dictionary]
@return: B{Properties}:
- tree: (dictionary) Object representing the new Reports tree
"""
# make sure we are not deleting a required node
if uid in essentialReportOrganizers:
raise Exception('You cannot delete this organizer')
# Getting all of the child nodes for auditing purposes
node = self.context.dmd.unrestrictedTraverse(uid)
brains = IModelCatalogTool(node).search((ReportClass,BaseReport))
family = []
for brain in brains:
family.append([brain.getPath(), isinstance(brain.getObject(), ReportClass)])
self._getFacade().deleteNode(uid)
# Audit messaging
for name, isOrganizer in family:
if isOrganizer:
audit('UI.Organizer.Delete', name)
else:
audit('UI.Report.Delete', name)
contextUid = '/'.join(uid.split('/')[:-1])
return self._getTreeUpdates(contextUid)
def filteredDevices(context, args, *types):
path = '/zport/dmd'
deviceFilter = args.get('deviceFilter', '') or ''
deviceClass = args.get('deviceClass', '') or ''
extraquery = args.get('extraquery', '')
filter = []
if deviceFilter:
filter.append(
MatchGlob('name', '*%s*' % deviceFilter)
| MatchGlob('id', '*%s*' % deviceFilter))
if deviceClass:
organizer = (''.join([path, deviceClass]), )
else:
organizer = (''.join(
[path, args.get('organizer', '/Devices') or '/Devices']), )
if not types:
types = 'Products.ZenModel.Device.Device'
if extraquery:
filter.extend(extraquery)
query = And(*filter) if filter else None
results = IModelCatalogTool(context).search(types,
paths=organizer,
query=query)
for brain in results:
try:
yield brain.getObject()
except Exception:
log.warn("Unable to unbrain at path %s", brain.getPath())
def __init__(self, dmd, event_summary):
self._dmd = dmd
self._event_summary = event_summary
self._eventOccurrence = event_summary['occurrence'][0]
self._eventActor = self._eventOccurrence['actor']
self._eventDetails = self._findDetails(self._eventOccurrence)
self._catalog = IModelCatalogTool(dmd)
self._manager = IGUIDManager(dmd)
def afterSetUp(self):
super(TestLayer2Linking, self).afterSetUp()
self.dev = self.dmd.Devices.createInstance("testdev")
manage_addIpInterface(self.dev.os.interfaces, 'eth0', True)
self.iface = self.dev.os.interfaces._getOb('eth0')
self.mac = '00:11:22:33:44:55'
self.iface._setPropValue('macaddress', self.mac)
self.catalog = IModelCatalogTool(self.dmd).layer2
def _search_device_catalog(self, query=None):
if USE_MODEL_CATALOG:
return IModelCatalogTool(self._dmd).devices.search(query=query)
else:
if query:
return self._dmd.Devices.deviceSearch.evalAdvancedQuery(query)
else:
return self._dmd.Devices.deviceSearch()
def __init__(self, context):
self.context = context
super(ModelQueryFacade, self).__init__(context)
self.model_catalog = IModelCatalogTool(self.context)
self.model_catalog_helper = ModelCatalogToolHelper(self.model_catalog)
self.indexed, self.stored, self.fields_by_type = self.model_catalog.model_catalog_client.get_indexes(
)
class CatalogTool(object):
implements(ICatalogTool)
def __init__(self, context):
self.context = context
self.model_catalog = IModelCatalogTool(self.context)
def getBrain(self, path):
return self.model_catalog.global_catalog.getBrain(path)
def parents(self, path):
return self.model_catalog.parents(path)
def count(self, types=(), path=None, filterPermissions=True):
return self.model_catalog.count(types=types,
path=path,
filterPermissions=filterPermissions)
def search(self,
types=(),
start=0,
limit=None,
orderby=None,
reverse=False,
paths=(),
depth=None,
query=None,
hashcheck=None,
filterPermissions=True,
globFilters=None):
return self.model_catalog.global_catalog.search(
types=types,
start=start,
limit=limit,
orderby=orderby,
reverse=reverse,
paths=paths,
depth=depth,
query=query,
hashcheck=hashcheck,
filterPermissions=filterPermissions,
globFilters=globFilters)
def update(self, obj):
self.model_catalog.update(obj)
def afterSetUp(self):
super(TestModelCatalogTransactions, self).afterSetUp()
# Lets change the ModelCatalogTestDataManager with ModelCatalogDataManager
self.model_catalog = IModelCatalogTool(self.dmd)
self.data_manager = ModelCatalogDataManager('localhost:8983', self.dmd)
self.model_catalog.model_catalog_client._data_manager = self.data_manager
# get a reference to model_index to be able to fo checks bypassing the data manager
self.model_index = self.data_manager.model_index
def _findDevices(self, identifier, ipAddress, limit=None):
"""
Returns a tuple ([device brains], [devices]) searching manage IP and
interface IPs. limit is the maximum total number in both lists.
"""
dev_cat = IModelCatalogTool(self._devices)
try:
ip_address = next(i for i in (ipAddress, identifier) if isip(i))
ip_decimal = ipToDecimal(ip_address)
except Exception:
ip_address = None
ip_decimal = None
quoted_id = quote_and_escape(identifier)
query_set = Or(Eq('id', quoted_id), Eq('name', quoted_id))
if ip_decimal is not None:
query_set.addSubquery(Eq('decimal_ipAddress', str(ip_decimal)))
device_brains = list(dev_cat.search(types=Device,
query=query_set,
limit=limit,
filterPermissions=False,
fields=["uid", "uuid"]))
if device_brains:
return device_brains, []
if ip_decimal is not None:
# don't search interfaces for 127.x.x.x IPv4 addresses
if ipToDecimal('126.255.255.255') < ip_decimal < ipToDecimal('128.0.0.0'):
ip_decimal = None
# don't search interfaces for the ::1 IPv6 address
elif ipToDecimal('::1') == ip_decimal:
ip_decimal = None
if ip_decimal is None:
return [], []
net_cat = IModelCatalogTool(self._networks)
results = net_cat.search(types=IpAddress,
query=(Eq('name', ip_address)),
limit = limit,
filterPermissions = False)
devices = [brain.getObject().device() for brain in results]
return device_brains, devices
def validate_templates(self):
""" Check that both catalogs return same data for templates """
global_catalog = ICatalogTool(dmd.Devices)
model_catalog = IModelCatalogTool(dmd.Devices)
# get template nodes from global catalog
global_catalog_brains = global_catalog.search(types=('Products.ZenModel.RRDTemplate.RRDTemplate',))
global_catalog_templates = set([ brain.getPath() for brain in global_catalog_brains ])
# get template nodes from model catalog
model_catalog_brains = global_catalog.search(types=('Products.ZenModel.RRDTemplate.RRDTemplate',))
model_catalog_templates = set([ brain.getPath() for brain in model_catalog_brains ])
# compare results
if len(model_catalog_templates - global_catalog_templates) == 0 and \
len(global_catalog_templates - model_catalog_templates) == 0:
for template in global_catalog_templates:
template_object = dmd.unrestrictedTraverse(template)
query = Eq('id', template_object.id)
gc_brains = global_catalog.search(types=('Products.ZenModel.RRDTemplate.RRDTemplate',), query=query)
gc_templates = set([ brain.getPath() for brain in gc_brains ])
mc_brains = model_catalog.search(types=('Products.ZenModel.RRDTemplate.RRDTemplate',), query=query)
mc_templates = set([ brain.getPath() for brain in mc_brains ])
failed_templates = []
if not (len(mc_templates - gc_templates) == 0 and \
len(gc_templates - mc_templates) == 0):
failed_templates.append(template)
if failed_templates:
print "TEST FAILED: Inconsistent results from catalogs for templates:"
for failure in failed_templates:
print "\t{0}".format(failure)
else:
print "TEST PASSED: Both catalogs returned same results!!"
return True
else:
print "TEST FAILED: Inconsistent results from catalogs:"
print "\t{0}".format("Templates found in global catalog and not in model catalog: {0}".format(global_catalog_templates - model_catalog_templates))
print "\t{0}".format("Templates found in model catalog and not in global catalog: {0}".format(model_catalog_templates - global_catalog_templates))
return False
def callHomeData(self, dmd):
self.dmd = dmd
self._catalog = IModelCatalogTool(self.dmd)
stats = (self.server_key, self.google_key, self.all_versions,
self.event_classes, self.event_count, self.reports,
self.templates, self.systems, self.groups, self.locations,
self.total_collectors, self.zenpacks, self.user_count,
self.product_count, self.product_name)
return chain.from_iterable(map(lambda fn: fn(), stats))
def __init__(self, ob, root=None, parent=None):
self._root = root or self
if getattr(self._root, '_ob_cache', None) is None:
self._root._ob_cache = {}
if not ICatalogBrain.providedBy(ob):
brain = IModelCatalogTool(ob).getBrainFromObject(ob)
if brain is None:
raise UncataloguedObjectException(ob)
# We already have the object - cache it here so _get_object doesn't
# have to look it up again.
self._root._ob_cache[brain.getPath()] = ob
ob = brain
self._object = ob
self._parent = parent or None
self._severity = None
# allow showing the event severity icons to be configurable
if not hasattr(self._root, '_showSeverityIcons'):
self._root._showSeverityIcons = self._shouldShowSeverityIcons()
def __init__(self, ob, root=None, parent=None):
self._root = root or self
if getattr(self._root, '_ob_cache', None) is None:
self._root._ob_cache = {}
if not ICatalogBrain.providedBy(ob):
brain = IModelCatalogTool(ob).getBrainFromObject(ob)
if brain is None:
raise UncataloguedObjectException(ob)
# We already have the object - cache it here so _get_object doesn't
# have to look it up again.
self._root._ob_cache[brain.getPath()] = ob
ob = brain
self._object = ob
self._parent = parent or None
self._severity = None
# allow showing the event severity icons to be configurable
if not hasattr(self._root, '_showSeverityIcons'):
self._root._showSeverityIcons = self._shouldShowSeverityIcons()
def processOrganizerUpdated(self, object, event):
catalog = IModelCatalogTool(object.primaryAq())
results = catalog.search(OSProcessClass)
if not results.total:
return
devices = set()
for organizer in results:
if results.areBrains:
organizer = organizer.getObject()
for process in organizer.instances():
device = process.device()
if not device:
continue
device = device.primaryAq()
device_path = device.getPrimaryUrlPath()
if not device_path in devices:
self._notifyAll(device)
devices.add(device_path)
def _get_brains(self, layer, attr, value):
"""
hack to make getLinkedNodes's awful code work with as little changes as possible
"""
model_catalog = IModelCatalogTool(self.dmd)
query = {}
if layer == 3:
model_catalog = model_catalog.layer3
meta_type = "IpAddress"
query["deviceId"] = "*" # We only are interested in assigned ips
else:
model_catalog = model_catalog.layer2
meta_type = "IpInterface"
query["meta_type"] = meta_type
if isinstance(value, basestring):
value = "*{0}".format(value)
query[attr] = value
search_results = model_catalog.search(query=query)
return [ brain for brain in search_results.results ]
def validate_mib_counts(self):
""" """
mib_organizers = get_mib_organizers(dmd)
failed_counts = []
global_catalog = ICatalogTool(dmd)
model_catalog = IModelCatalogTool(dmd)
for organizer in mib_organizers:
global_catalog_count = global_catalog.count(("Products.ZenModel.MibModule.MibModule",), organizer)
model_catalog_count = model_catalog.count(("Products.ZenModel.MibModule.MibModule",), organizer)
if global_catalog_count != model_catalog_count:
failed_counts.append(organizer)
if not failed_counts:
print "TEST PASSED: All mib organizers have the same count in both catalogs!!"
else:
print "TEST FAILED: The following mib organizers have different counts in the catalogs:"
for failed in failed_counts:
print "\t{0}".format(failed)
return len(failed_counts) == 0
def __call__(self, query='', dataRoot='devices'):
"""
@param query: A glob by which to filter device names
@type query: str
@return: A JSON representation of a list of ids
@rtype: "['id1', 'id2', 'id3']"
"""
if dataRoot != 'devices':
import exceptions
raise exceptions.ValueError("dataRoot should only be 'devices'")
query_scope = self.context.dmd.Devices
query = MatchGlob('name', query.rstrip('*') + '*')
if isinstance(self.context, DeviceOrganizer):
query_scope = self.context
catalog = IModelCatalogTool(query_scope).devices
brains = catalog.search(query=query, fields=['name'])
return sorted((b.name for b in brains),
key=lambda x: x.lower())
def _getSearchResultsFromModelCatalog(self, parsedQuery, sorter=None, category=None, countOnly=False,
unrestricted=False, filterFn=None, maxResults=None):
operators = parsedQuery.operators
keywords = parsedQuery.keywords
if not keywords:
return
def listMatchGlob(op, index, list):
return op(*[MatchGlob(index, '*%s*' % i) for i in list])
dmd = self._dmd
kw_query = Or(listMatchGlob(And, 'name', keywords),
listMatchGlob(And, 'text_ipAddress', keywords))
# Rank devices whose name match the query higher than other stuff
# TODO: Figure out how to expose Lucene boosts
# For now we just or the boost query in with the original query to boost those results
ranker = listMatchGlob(Or, 'name', keywords)
full_query = Or(kw_query, ranker)
cat = IModelCatalogTool(dmd).devices
limit = 0 if countOnly and not filterFn else maxResults
# Set orderby to None so that modelindex will rank by score
catalogItems = cat.search(query=full_query, orderby=None, filterPermissions=True, limit=limit)
brainResults = [DeviceSearchResult(catalogItem) for catalogItem in catalogItems]
if countOnly and not filterFn:
return dict(Device=brainResults.total)
if filterFn:
brainResults = filter(filterFn, brainResults)
if countOnly:
return dict(Device=len(brainResults))
results = brainResults
if sorter is not None:
results = sorter.limitSort(results)
return results
def validate_device_components(self):
""" search for devices components with both old a new catalogs """
model_catalog = IModelCatalogTool(dmd)
failed_devices = []
object_implements_query = Eq('objectImplements', "Products.ZenModel.DeviceComponent.DeviceComponent")
for device in self._get_all_devices():
device_catalog = device.componentSearch
device_catalog_components = defaultdict(list)
for brain in device_catalog():
device_catalog_components[brain.meta_type].append(brain.getPath())
device_uid = "/".join(device.getPrimaryPath())
model_query = And(object_implements_query, Eq("deviceId", device_uid))
model_query_results = model_catalog.search(query=model_query)
model_catalog_components = defaultdict(list)
for brain in model_query_results.results:
model_catalog_components[brain.meta_type].append(brain.getPath())
same_keys = len(device_catalog_components.keys()) == len(model_catalog_components.keys()) and \
len(set(device_catalog_components.keys()) - set(model_catalog_components.keys())) == 0
if not same_keys:
failed_devices.append(device)
else:
for meta_type in model_catalog_components.keys():
device_catalog_values = device_catalog_components[meta_type]
model_catalog_values = model_catalog_components[meta_type]
same_values = len(device_catalog_values) == len(model_catalog_values) and \
len(set(device_catalog_values) - set(model_catalog_values)) == 0
if not same_values:
failed_devices.append(device)
break
if failed_devices:
print "TEST FAILED: Catalogs return different components for the following devices:"
for dev in failed_devices:
print "\t\t{0}".format(dev.getPrimaryId())
else:
print "TEST PASSED: Both catalogs returned the same components for every device."
return len(failed_devices) == 0
