def test(self, dmd):
from Products.Zuul.catalog.interfaces import IModelCatalogTool
results = IModelCatalogTool(dmd.Devices).search(
'Products.ZenModel.DeviceOrganizer.DeviceOrganizer')
instances = IModelCatalogTool(
dmd.Devices).search('Products.ZenModel.Device.Device')
tree = PathIndexCache(results, instances, 'devices')
print tree
def _search_catalog(self, obj, types=(), paths=(), query=None):
if USE_MODEL_CATALOG:
catalog = IModelCatalogTool(obj)
else:
catalog = ICatalogTool(obj)
return catalog.search(types=types, paths=paths, query=query)
def filteredDevices(context, args, *types):
path = '/zport/dmd'
deviceFilter = args.get('deviceFilter', '') or ''
deviceClass = args.get('deviceClass', '') or ''
extraquery = args.get('extraquery', '')
filter = []
if deviceFilter:
filter.append(
MatchGlob('name', '*%s*' % deviceFilter)
| MatchGlob('id', '*%s*' % deviceFilter))
if deviceClass:
organizer = (''.join([path, deviceClass]), )
else:
organizer = (''.join(
[path, args.get('organizer', '/Devices') or '/Devices']), )
if not types:
types = 'Products.ZenModel.Device.Device'
if extraquery:
filter.extend(extraquery)
query = And(*filter) if filter else None
results = IModelCatalogTool(context).search(types,
paths=organizer,
query=query)
for brain in results:
try:
yield brain.getObject()
except Exception:
log.warn("Unable to unbrain at path %s", brain.getPath())
def cutover(self, dmd):
log.info("Updating relationships on Ip Addresses")
catalog = IModelCatalogTool(dmd.Networks)
for ip in catalog.search(IpAddress):
try:
ip.getObject().buildRelations()
#some objects can fail relations building process
except (Exception, ):
continue
log.info("Updating relationships on Devices")
for dev in dmd.Devices.getSubDevices():
try:
dev.buildRelations()
if dev.manageIp:
ipobj = dev.getNetworkRoot().findIp(dev.manageIp)
if ipobj:
dev.ipaddress.addRelation(ipobj)
else:
ipWithoutNetmask, netmask = ipAndMaskFromIpMask(
dev.manageIp)
ipobj = dev.getNetworkRoot().createIp(
ipWithoutNetmask, netmask)
dev.ipaddress.addRelation(ipobj)
notify(IndexingEvent(ipobj))
#some objects can fail relations building process
except (Exception, ):
continue
def moveProcess(self, uid, targetUid):
obj = self._getObject(uid)
target = self._getObject(targetUid)
brainsCollection = []
# reindex all the devices and processes underneath this guy and the target
for org in (obj.getPrimaryParent().getPrimaryParent(), target):
catalog = IModelCatalogTool(org)
brainsCollection.append(catalog.search(OSProcess))
if isinstance(obj, OSProcessClass):
source = obj.osProcessOrganizer()
source.moveOSProcessClasses(targetUid, obj.id)
newObj = getattr(target.osProcessClasses, obj.id)
elif isinstance(obj, OSProcessOrganizer):
source = aq_parent(obj)
source.moveOrganizer(targetUid, (obj.id, ))
newObj = getattr(target, obj.id)
else:
raise Exception('Illegal type %s' % obj.__class__.__name__)
# fire the object moved event for the process instances (will update catalog)
for brains in brainsCollection:
objs = imap(unbrain, brains)
for item in objs:
notify(
ObjectMovedEvent(item, item.os(), item.id, item.os(),
item.id))
return newObj.getPrimaryPath()
def deleteNode(self, uid):
"""
Remove a report or report organizer.
@type uid: string
@param uid: The UID of the node to delete
@rtype: [dictionary]
@return: B{Properties}:
- tree: (dictionary) Object representing the new Reports tree
"""
# make sure we are not deleting a required node
if uid in essentialReportOrganizers:
raise Exception('You cannot delete this organizer')
# Getting all of the child nodes for auditing purposes
node = self.context.dmd.unrestrictedTraverse(uid)
brains = IModelCatalogTool(node).search((ReportClass,BaseReport))
family = []
for brain in brains:
family.append([brain.getPath(), isinstance(brain.getObject(), ReportClass)])
self._getFacade().deleteNode(uid)
# Audit messaging
for name, isOrganizer in family:
if isOrganizer:
audit('UI.Organizer.Delete', name)
else:
audit('UI.Report.Delete', name)
contextUid = '/'.join(uid.split('/')[:-1])
return self._getTreeUpdates(contextUid)
def _getDeviceBatch(self,
selectstatus='none',
goodevids=[],
badevids=[],
offset=0,
count=50,
filter='',
orderby='titleOrId',
orderdir='asc'):
unused(count, offset, orderby, orderdir)
if not isinstance(goodevids, (list, tuple)):
goodevids = [goodevids]
if not isinstance(badevids, (list, tuple)):
badevids = [badevids]
if selectstatus == 'all':
idquery = ~In('id', badevids)
else:
idquery = In('id', goodevids)
devfilter = '(?is).*%s.*' % filter
filterquery = Or(MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter))
query = Eq('uid', self.context.absolute_url_path()) & idquery
query = query & filterquery
catalog = IModelCatalogTool(self.context)
objects = catalog.search(query=query)
return [x['id'] for x in objects]
def _find_instances_in_catalog(self, count=False):
model_catalog = IModelCatalogTool(self.dmd)
query = {"productClassId": self.idx_uid()}
if count:
return model_catalog.search(query=query, limit=0)
else:
return model_catalog.search(query=query)
def _getAdvancedQueryDeviceList(self,
offset=0,
count=50,
filter='',
orderby='name',
orderdir='asc'):
"""
Ask the catalog for devices matching the criteria specified.
"""
context = self.context
if not isinstance(context, DeviceOrganizer):
context = self.context.dmd.Devices
catalog = IModelCatalogTool(context).devices
devfilter = '(?is).*%s.*' % filter
filterquery = Or(MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter))
query = Eq('uid', context.absolute_url_path()) & filterquery
objects = catalog.search(query=query,
start=int(offset),
limit=int(count),
order_by=orderby,
reverse=orderdir != 'asc')
objects = list(objects)
totalCount = len(objects)
return totalCount, objects
def testDeleteOrganizerRemovesDevices(self):
"""When we delete an organizer all the devices assigned to it should not
still have a relationship to it in the catalog
"""
# Create Organizer
organizer = self.facade.addOrganizer("/zport/dmd/Groups",
'testOrganizer')
organizer_path = organizer.uid
catalog = IModelCatalogTool(self.dmd)
# Add some devices to it (use createInstance to create a device)
devices = self.dmd.Devices
test_device = devices.createInstance('testDevice')
self.facade.moveDevices(['/'.join(test_device.getPhysicalPath())],
organizer_path)
organizer = self.dmd.unrestrictedTraverse(organizer_path)
# test we have added the device
self.assertEqual(len(organizer.getDevices()), 1,
"make sure we saved our device")
deviceBrains = catalog.search(
paths='/'.join(organizer.getPhysicalPath()))
self.assertTrue(
deviceBrains.total > 1,
" At this point we should have the organizer and the device")
# Delete the Organizer
self.facade.deleteNode(organizer_path)
# Get the devices directly from the path
deviceBrains = catalog.search(
paths='/'.join(organizer.getPhysicalPath()))
self.assertEqual(deviceBrains.total, 0,
" we should not have any devices at this point")
def getAddTemplateTargets(self, query=None):
"""
@returns list of targets for our new template
"""
cat = IModelCatalogTool(self._dmd)
results = []
# it can be any device class that we add the template too
brains = cat.search(types=[DeviceClass])
for brain in brains:
# HACK: use the path to get the organizer name so we don't have to wake up the object
label = brain.getPath()
if label == "/zport/dmd/Devices":
label = "/"
else:
label = label.replace("/zport/dmd/Devices/", "/")
results.append(dict(uid=brain.getPath(), label=label))
# the display is organizer name and we do not index it
# so filter after the query
if query is not None:
results = [
result for result in results
if query.lower() in result['label'].lower()
]
return sorted(results, key=lambda org: org['label'])
def _findDevices(self, identifier, ipAddress, limit=None):
"""
Returns a tuple ([device brains], [devices]) searching manage IP and
interface IPs. limit is the maximum total number in both lists.
"""
dev_cat = IModelCatalogTool(self._devices)
try:
ip_address = next(i for i in (ipAddress, identifier) if isip(i))
ip_decimal = ipToDecimal(ip_address)
except Exception:
ip_address = None
ip_decimal = None
quoted_id = quote_and_escape(identifier)
query_set = Or(Eq('id', quoted_id), Eq('name', quoted_id))
if ip_decimal is not None:
query_set.addSubquery(Eq('decimal_ipAddress', str(ip_decimal)))
device_brains = list(
dev_cat.search(types=Device,
query=query_set,
limit=limit,
filterPermissions=False,
fields=["uid", "uuid"]))
if device_brains:
return device_brains, []
if ip_decimal is not None:
# don't search interfaces for 127.x.x.x IPv4 addresses
if ipToDecimal('126.255.255.255') < ip_decimal < ipToDecimal(
'128.0.0.0'):
ip_decimal = None
# don't search interfaces for the ::1 IPv6 address
elif ipToDecimal('::1') == ip_decimal:
ip_decimal = None
if ip_decimal is None:
return [], []
net_cat = IModelCatalogTool(self._networks)
results = net_cat.search(types=IpAddress,
query=(Eq('name', ip_address)),
limit=limit,
filterPermissions=False)
devices = [brain.getObject().device() for brain in results]
return device_brains, devices
def __init__(self, dmd, event_summary):
self._dmd = dmd
self._event_summary = event_summary
self._eventOccurrence = event_summary['occurrence'][0]
self._eventActor = self._eventOccurrence['actor']
self._eventDetails = self._findDetails(self._eventOccurrence)
self._catalog = IModelCatalogTool(dmd)
self._manager = IGUIDManager(dmd)
def afterSetUp(self):
super(TestModelCatalogTransactions, self).afterSetUp()
# Lets change the ModelCatalogTestDataManager with ModelCatalogDataManager
self.model_catalog = IModelCatalogTool(self.dmd)
self.data_manager = ModelCatalogDataManager('localhost:8983', self.dmd)
self.model_catalog.model_catalog_client._data_manager = self.data_manager
# get a reference to model_index to be able to fo checks bypassing the data manager
self.model_index = self.data_manager.model_index
def _search_ip_in_catalog(self, ip):
""" Return list of brains that match ip """
cat = IModelCatalogTool(self.getNetworkRoot())
query = {}
query["objectImplements"] = "Products.ZenModel.IpAddress.IpAddress"
query["id"] = ipwrap(ip)
search_result = cat.search(query=query)
return [brain for brain in search_result.results]
def _search_device_catalog(self, query=None):
if USE_MODEL_CATALOG:
return IModelCatalogTool(self._dmd).devices.search(query=query)
else:
if query:
return self._dmd.Devices.deviceSearch.evalAdvancedQuery(query)
else:
return self._dmd.Devices.deviceSearch()
def afterSetUp(self):
super(TestLayer2Linking, self).afterSetUp()
self.dev = self.dmd.Devices.createInstance("testdev")
manage_addIpInterface(self.dev.os.interfaces, 'eth0', True)
self.iface = self.dev.os.interfaces._getOb('eth0')
self.mac = '00:11:22:33:44:55'
self.iface._setPropValue('macaddress', self.mac)
self.catalog = IModelCatalogTool(self.dmd).layer2
def __init__(self, context):
self.context = context
super(ModelQueryFacade, self).__init__(context)
self.model_catalog = IModelCatalogTool(self.context)
self.model_catalog_helper = ModelCatalogToolHelper(self.model_catalog)
self.indexed, self.stored, self.fields_by_type = self.model_catalog.model_catalog_client.get_indexes(
)
def _get_devices_under_path(self, path):
""" Returnd device's brains """
model_catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}".format(path)
fields = ["id", "path"]
result = model_catalog.search(query=query, fields=fields)
return result.results
def callHomeData(self, dmd):
self.dmd = dmd
self._catalog = IModelCatalogTool(self.dmd)
stats = (self.server_key, self.google_key, self.all_versions,
self.event_classes, self.event_count, self.reports,
self.templates, self.systems, self.groups, self.locations,
self.total_collectors, self.zenpacks, self.user_count,
self.product_count, self.product_name)
return chain.from_iterable(map(lambda fn: fn(), stats))
def _getSubdevices(self, brains=False):
""" Helper method to search for devices/devices brains under the current organizer """
catalog = IModelCatalogTool(self.dmd.Devices)
query = {}
query["objectImplements"] = "Products.ZenModel.Device.Device"
query["path"] = "{0}*".format("/".join(self.getPhysicalPath()))
if not brains:
return getObjectsFromModelCatalog(catalog, query, LOG)
else:
return catalog.search(query=query).results
def _get_component_brains_from_model_catalog(self, uid, meta_type=()):
""" """
model_catalog = IModelCatalogTool(self.context.dmd)
query = {}
if meta_type:
query["meta_type"] = meta_type
query["objectImplements"] = "Products.ZenModel.DeviceComponent.DeviceComponent"
query["deviceId"] = uid
model_query_results = model_catalog.search(query=query)
brains = [ brain for brain in model_query_results.results ]
return brains
def clearGeocodeCache(self):
"""
This clears the geocode cache by reseting the latlong property of
all locations.
"""
results = IModelCatalogTool(self._dmd.Locations).search('Products.ZenModel.Location.Location')
for brain in results:
try:
brain.getObject().latlong = None
except Exception:
log.warn("Unable to clear the geocodecache from %s " % brain.getPath())
def initialize(self, context):
root = self.getRoot(context)
brains = IModelCatalogTool(root).search(self._types)
results = {}
for brain in brains:
try:
obj = brain.getObject()
results[brain.getPath()] = self.organizerChecksum(obj)
except KeyError:
log.warn("Unable to retrieve object: %s", brain.getPath())
self.checksum_map = results
def findDeviceByIdExact(self, devicename):
"""
Look up device in catalog and return it. devicename
must match device id exactly
"""
if devicename:
query = And( Eq("objectImplements", "Products.ZenModel.Device.Device"), Eq('id', devicename) )
search_results = IModelCatalogTool(self.dmd.Devices).search(query=query)
for brain in search_results.results:
dev = brain.getObject()
if dev.id == devicename:
return dev
def remote_createAllUsers(self):
cat = IModelCatalogTool(self.dmd)
brains = cat.search(("Products.ZenModel.Device.Device", "Products.ZenModel.DeviceClass.DeviceClass"))
users = []
for brain in brains:
device = brain.getObject()
user = self._create_user(device)
if user is not None:
users.append(user)
fmt = 'SnmpTrapConfig.remote_createAllUsers {0} users'
log.debug(fmt.format(len(users)))
return users
def _getSearchResultsFromModelCatalog(self,
parsedQuery,
sorter=None,
category=None,
countOnly=False,
unrestricted=False,
filterFn=None,
maxResults=None):
operators = parsedQuery.operators
keywords = parsedQuery.keywords
if not keywords:
return
def listMatchGlob(op, index, list):
return op(*[MatchGlob(index, '*%s*' % i) for i in list])
dmd = self._dmd
kw_query = Or(listMatchGlob(And, 'name', keywords),
listMatchGlob(And, 'text_ipAddress', keywords))
# Rank devices whose name match the query higher than other stuff
# TODO: Figure out how to expose Lucene boosts
# For now we just or the boost query in with the original query to boost those results
ranker = listMatchGlob(Or, 'name', keywords)
full_query = Or(kw_query, ranker)
cat = IModelCatalogTool(dmd).devices
limit = 0 if countOnly and not filterFn else maxResults
# Set orderby to None so that modelindex will rank by score
catalogItems = cat.search(query=full_query,
orderby=None,
filterPermissions=True,
limit=limit)
brainResults = [
DeviceSearchResult(catalogItem) for catalogItem in catalogItems
]
if countOnly and not filterFn:
return dict(Device=brainResults.total)
if filterFn:
brainResults = filter(filterFn, brainResults)
if countOnly:
return dict(Device=len(brainResults))
results = brainResults
if sorter is not None:
results = sorter.limitSort(results)
return results
def getSubComponents(self):
cat = IModelCatalogTool(self)
# @TODO: Can we avoid NOTs ?
query = Eq('objectImplements',
'Products.ZenModel.DeviceComponent.DeviceComponent')
brains = cat.search(query=query)
children = []
for brain in brains:
try:
children.append(brain.getObject())
except:
pass
return children
def getObjectBrains(self,
uid=None,
start=0,
limit=50,
sort='name',
dir='ASC',
params=None,
hashcheck=None,
types=(),
fields=[]):
cat = IModelCatalogTool(self._getObject(uid))
reverse = bool(dir == 'DESC')
qs = []
query = None
globFilters = {}
prodStates = None
params = params if params else {}
for key, value in params.iteritems():
if key == 'ipAddress':
qs.append(MatchGlob('text_ipAddress', '{}*'.format(value)))
elif key == 'productionState':
qs.append(
Or(*[Eq('productionState', str(state))
for state in value]))
# ZEN-30949 - stringify values from the 'priority' list if it's passed in for query criteria
elif key == 'priority':
qs.append(
Or(*[Eq('priority', str(priority)) for priority in value]))
# ZEN-10057 - move filtering on indexed groups/systems/location from post-filter to query
elif key in organizersToClass:
organizerQuery = self.findMatchingOrganizers(
organizersToClass[key], organizersToPath[key], value)
if not organizerQuery:
return []
qs.append(organizerQuery)
else:
globFilters[key] = value
if qs:
query = And(*qs)
return cat.search(types,
start=start,
limit=limit,
orderby=sort,
reverse=reverse,
query=query,
globFilters=globFilters,
hashcheck=hashcheck,
fields=fields)
def cutover(self, dmd):
search_results = IModelCatalogTool(dmd).devices.search(
limit=1, fields=["snmpLastCollection"])
if search_results.total > 0 and search_results.results.next(
).snmpLastCollection is None:
log.info(
"Adding index for last model time, this can take a while.")
reindex_model_catalog(dmd,
root="/zport/dmd/Devices",
idxs=("snmpLastCollection"),
types=DeviceIndexable)
else:
log.info("Index for last model time already exists, skipping.")
