def searchForMembers(self, REQUEST=None, **kw): """Search for users (not groups, not groups-as-users) of a site """ if REQUEST: dict = REQUEST else: dict = kw name = dict.get('name', '') # Split up search terms but leave quoted ones together try: names = shlex.split(name) except ValueError: try: names = shlex.split(name.replace("'", "\\'")) except ValueError: names = shlex.split(name.replace("'", "\\'") + '"') # Short circuit: if all that was asked for was '*', just return everyone if names == ['*']: query = And() else: queries = [] for name in names: queries.extend([ MatchGlob('fullname', name), MatchGlob('email', name), Eq('getUserName', name) ]) query = Or(*queries) zLOG.LOG('MembershipTool', zLOG.BLATHER, 'Querying: %s' % query) catalog = getToolByName(self, 'member_catalog') return catalog.evalAdvancedQuery(query, ('surname', 'firstname'))
def filteredDevices(context, args, *types): path = '/zport/dmd' deviceFilter = args.get('deviceFilter', '') or '' deviceClass = args.get('deviceClass', '') or '' extraquery = args.get('extraquery', '') filter = [] if deviceFilter: filter.append( MatchGlob('name', '*%s*' % deviceFilter) | MatchGlob('id', '*%s*' % deviceFilter)) if deviceClass: organizer = (''.join([path, deviceClass]), ) else: organizer = (''.join( [path, args.get('organizer', '/Devices') or '/Devices']), ) if not types: types = 'Products.ZenModel.Device.Device' if extraquery: filter.extend(extraquery) query = And(*filter) if filter else None results = IModelCatalogTool(context).search(types, paths=organizer, query=query) for brain in results: try: yield brain.getObject() except Exception: log.warn("Unable to unbrain at path %s", brain.getPath())
def testSearches(self): n_organizers = 100 organizers = {} pattern = "testSearches_DEVICE_CLASS_" for i in xrange(n_organizers): dc = self.dmd.Devices.createOrganizer("{}{:02}".format(pattern, i)) organizers[dc.idx_uid()] = dc query = MatchGlob(UID, "/zport/dmd/Devices/{}*".format(pattern)) search_results = self.model_catalog.search(query=query, commit_dirty=False) self.assertTrue(search_results.total == 0) search_results = self.model_catalog.search(query=query, commit_dirty=True) self.assertTrue(search_results.total == n_organizers) search_results = self.model_catalog.search(query=query, limit=0) self.assertTrue(search_results.total == n_organizers) limit = 18 for start in [0, 12, 45, 70]: expected_uids = { "/zport/dmd/Devices/{}{:02}".format(pattern, i) for i in xrange(start, start + limit) } search_results = self.model_catalog.search(query=query, start=start, limit=limit) self.assertTrue(search_results.total == n_organizers) brain_uids = { getattr(brain, UID) for brain in search_results.results } self.assertEquals(len(brain_uids), limit) self.assertEquals(len(brain_uids), len(expected_uids)) self.assertTrue(len(expected_uids - brain_uids) == 0) self.assertTrue(len(brain_uids - expected_uids) == 0)
def _typecatComponentBrains(self, uid=None, types=(), meta_type=(), start=0, limit=None, sort='name', dir='ASC', name=None, keys=()): obj = self._getObject(uid) spec = get_component_field_spec(meta_type) if spec is None: return None, 0 typecat = spec.get_catalog(obj, meta_type) sortspec = () if sort: if sort not in typecat._catalog.indexes: # Fall back to slow queries and sorting return None, 0 sortspec = ((sort, dir), ) querySet = [Generic('path', uid)] if name: querySet.append( Or(*(MatchGlob(field, '*%s*' % name) for field in spec.fields))) brains = typecat.evalAdvancedQuery(And(*querySet), sortspec) total = len(brains) if limit is None: brains = brains[start:] else: brains = brains[start:start + limit] return brains, total
def get_uids(index_client, root="", types=()): start = 0 need_results = True query = [Eq("tx_state", 0)] if root: root = root.rstrip('/') query.append( Or(Eq("uid", "{}".format(root)), MatchGlob("uid", "{}/*".format(root)))) if not isinstance(types, (tuple, list)): types = (types, ) if types: query.append(In("objectImplements", [dottedname(t) for t in types])) while need_results: search_results = index_client.search( SearchParams(query=And(*query), start=start, limit=MODEL_INDEX_BATCH_SIZE, order_by="uid", fields=["uid"])) start += MODEL_INDEX_BATCH_SIZE for result in search_results.results: yield result.uid need_results = start < search_results.total_count
def _findDevice(self, devicename, useTitle=True, commit_dirty=False): """ Returns all devices whose ip/id/title match devicename. ip/id matches are at the front of the list. @rtype: list of brains """ ors = [ MatchGlob('id', devicename), MatchGlob('text_ipAddress', devicename) ] if useTitle: ors.append(MatchGlob('name', devicename)) query = And( Eq("objectImplements", "Products.ZenModel.Device.Device"), Or(*ors) ) fields = [ "name", "id", "text_ipAddress" ] search_results = IModelCatalogTool(self.dmd.Devices).search(query=query, fields=fields, commit_dirty=commit_dirty) return list(search_results.results)
def _findDevice(self, devicename, useTitle=True): """ Returns all devices whose ip/id/title match devicename. ip/id matches are at the front of the list. @rtype: list of brains """ idIpQuery = Or(MatchGlob('id', devicename), Eq('getDeviceIp', devicename)) if useTitle: titleOrIdQuery = MatchGlob('titleOrId', devicename) query = Or(idIpQuery, titleOrIdQuery) rankSort = RankByQueries_Max((idIpQuery, 16), (titleOrIdQuery, 8)) devices = self._getCatalog().evalAdvancedQuery(query, (rankSort, )) else: devices = self._getCatalog().evalAdvancedQuery(idIpQuery) return devices
def devices(self, name): """return managed devices from ip, name or mac address taken from ZentinelPortal.py""" zcatalog = self.dmd.Devices.deviceSearch glob = name.rstrip('*') + '*' glob = MatchGlob('id', glob) query = Or(glob, Eq('getDeviceIp', name)) brains = zcatalog.evalAdvancedQuery(query) brains += self.dmd.Networks.ipSearch.evalAdvancedQuery(glob) return [b.getObject() for b in brains]
def __call__(self, query='', dataRoot='devices'): """ @param query: A glob by which to filter device names @type query: str @return: A JSON representation of a list of ids @rtype: "['id1', 'id2', 'id3']" """ if dataRoot != 'devices': import exceptions raise exceptions.ValueError("dataRoot should only be 'devices'") query_scope = self.context.dmd.Devices query = MatchGlob('name', query.rstrip('*') + '*') if isinstance(self.context, DeviceOrganizer): query_scope = self.context catalog = IModelCatalogTool(query_scope).devices brains = catalog.search(query=query, fields=['name']) return sorted((b.name for b in brains), key=lambda x: x.lower())
def __call__(self, query='', dataRoot='devices'): """ @param query: A glob by which to filter device names @type query: str @return: A JSON representation of a list of ids @rtype: "['id1', 'id2', 'id3']" """ if dataRoot != 'devices': import exceptions raise exceptions.ValueError("dataRoot should only be 'devices'") catalog = getToolByName(self.context.dmd.Devices, self.context.dmd.Devices.default_catalog) query = MatchGlob('titleOrId', query.rstrip('*') + '*') if isinstance(self.context, DeviceOrganizer): query = query & Eq('path', "/".join(self.context.getPhysicalPath())) brains = catalog.evalAdvancedQuery(query) # TODO: Add titleOrId to the catalog's metadata. return sorted((b.getObject().titleOrId() for b in brains), key=lambda x: x.lower())
def __call__(self, query='', dataRoot='devices'): """ @param query: A glob by which to filter device names @type query: str @return: A JSON representation of a list of ids @rtype: "['id1', 'id2', 'id3']" """ if dataRoot != 'devices': import exceptions raise exceptions.ValueError("dataRoot should only be 'devices'") catalog = getToolByName(self.context.dmd.Devices, self.context.dmd.Devices.default_catalog) query = MatchGlob('titleOrId', query.rstrip('*') + '*') if isinstance(self.context, DeviceOrganizer): query = query & Eq('path', "/".join(self.context.getPhysicalPath())) brains = catalog.evalAdvancedQuery(query) # TODO: Add titleOrId to the catalog's metadata. return sorted((b.getObject().titleOrId() for b in brains), key=lambda x: x.lower())
def __call__(self, query='', dataRoot='devices'): """ @param query: A glob by which to filter device names @type query: str @return: A JSON representation of a list of ids @rtype: "['id1', 'id2', 'id3']" """ if dataRoot != 'devices': import exceptions raise exceptions.ValueError("dataRoot should only be 'devices'") query_scope = self.context.dmd.Devices query = MatchGlob('name', query.rstrip('*') + '*') if isinstance(self.context, DeviceOrganizer): query_scope = self.context catalog = IModelCatalogTool(query_scope).devices brains = catalog.search(query=query, fields=['name']) return sorted((b.name for b in brains), key=lambda x: x.lower())
def searchDevices(self, queryString='', REQUEST=None): """Returns the concatenation of a device name, ip and mac search on the list of devices. """ # TODO: Remove. Not used anymore in Zenoss code --Ian zcatalog = self.dmd.Devices.deviceSearch glob = queryString.rstrip('*') + '*' idGlob = MatchGlob('id', glob) titleGlob = MatchGlob('titleOrId', glob) idOrTitleQuery = Or(idGlob,titleGlob) query = Or(idOrTitleQuery, Eq('getDeviceIp', queryString)) additionalQuery = self._additionalQuery() if additionalQuery: query = And( query, additionalQuery ) brains = zcatalog.evalAdvancedQuery(query) if REQUEST and len(brains) == 1: raise Redirect(urllib.quote(brains[0].getPrimaryId)) if additionalQuery: idGlob = And( idGlob, additionalQuery ) brains += self.dmd.Networks.ipSearch.evalAdvancedQuery(idGlob) return [ b.getObject() for b in brains ]
def getObjectBrains(self, uid=None, start=0, limit=50, sort='name', dir='ASC', params=None, hashcheck=None, types=(), fields=[]): cat = IModelCatalogTool(self._getObject(uid)) reverse = bool(dir == 'DESC') qs = [] query = None globFilters = {} prodStates = None params = params if params else {} for key, value in params.iteritems(): if key == 'ipAddress': qs.append(MatchGlob('text_ipAddress', '{}*'.format(value))) elif key == 'productionState': qs.append( Or(*[Eq('productionState', str(state)) for state in value])) # ZEN-30949 - stringify values from the 'priority' list if it's passed in for query criteria elif key == 'priority': qs.append( Or(*[Eq('priority', str(priority)) for priority in value])) # ZEN-10057 - move filtering on indexed groups/systems/location from post-filter to query elif key in organizersToClass: organizerQuery = self.findMatchingOrganizers( organizersToClass[key], organizersToPath[key], value) if not organizerQuery: return [] qs.append(organizerQuery) else: globFilters[key] = value if qs: query = And(*qs) return cat.search(types, start=start, limit=limit, orderby=sort, reverse=reverse, query=query, globFilters=globFilters, hashcheck=hashcheck, fields=fields)
def get_rrd_templates(self, *args, **kwargs): query = Eq("objectImplements", "Products.ZenModel.RRDTemplate.RRDTemplate") kwargs["query"] = And(query, MatchGlob('uid', "/zport/dmd/Devices*")) return self.model_catalog.search(*args, **kwargs)
def testDataManager(self): # before any changes are made, tx_state is None self.assertIsNone(self._get_transaction_state()) device_class_1 = "device_class_1" device_class_2 = "device_class_2" device_class_3 = "device_class_3" device_class_4 = "device_class_4" # create an organizer dc_1 = self.dmd.Devices.createOrganizer(device_class_1) tx_state = self._get_transaction_state() dc_1_uid = dc_1.idx_uid() # Some tx_state checks self.assertIsNotNone(tx_state) self.assertTrue(len(tx_state.pending_updates) > 0) self.assertTrue(len(tx_state.indexed_updates) == 0) self.assertTrue(len(tx_state.temp_indexed_uids) == 0) self.assertTrue(len(tx_state.temp_deleted_uids) == 0) # The new organizer index update should have been buffered in tx_state self._check_tx_state(pending=dc_1_uid) # A search with commit_dirty=False should not find the new device organizer search_results = self.model_catalog.search(query=Eq(UID, dc_1_uid), commit_dirty=False) self.assertEquals(search_results.total, 0) # A search with commit_dirty=True must find the new device organizer search_results = self.model_catalog.search(query=Eq(UID, dc_1_uid), commit_dirty=True) # model catalog should return the dirty doc self.assertEquals(search_results.total, 1) self._validate_temp_indexed_results(search_results, expected_object_uids=[dc_1_uid]) # the tx_state object should have been updated appropiately self._check_tx_state(temp_indexed=dc_1_uid) self.assertTrue(len(tx_state.pending_updates) == 0) # create another organizer dc_2 = self.dmd.Devices.createOrganizer(device_class_2) dc_2_uid = dc_2.idx_uid() # check tx_state has been updated accordinly self._check_tx_state(pending=dc_2_uid, temp_indexed=dc_1_uid) # search for both device classes with commit_dirty=False, it should only return dc_1_uid query = MatchGlob(UID, "/zport/dmd/Devices/device_class*") search_results = self.model_catalog.search(query=query, commit_dirty=False) self._validate_temp_indexed_results(search_results, expected_object_uids=[dc_1_uid]) # tx_state should not have changed self._check_tx_state(pending=dc_2_uid, temp_indexed=dc_1_uid) # now with commit_dirty=True search_results = self.model_catalog.search(query=query, commit_dirty=True) self._check_tx_state(temp_indexed=[dc_1_uid, dc_2_uid]) # it should return 2 device classes self.assertEquals(search_results.total, 2) self._validate_temp_indexed_results( search_results, expected_object_uids=[dc_1_uid, dc_2_uid]) # Lets delete device_class_1 self.dmd.Devices._delObject(device_class_1) self._check_tx_state(pending=[dc_1_uid]) # a search with commit = True should not return device_class_1 anymore search_results = self.model_catalog.search(query=query, commit_dirty=True) self._validate_temp_indexed_results(search_results, expected_object_uids=[dc_2_uid]) self._check_tx_state(temp_deleted=[dc_1_uid]) # however, we should have two temp docs matching "/zport/dmd/Devices/device_class*" mi_results = self.model_index.search(SearchParams(query)) self.assertTrue(mi_results.total_count == 2) # make sure a count type of query works (search with limit=0) search_results = self.model_catalog.search(query=query, limit=0, commit_dirty=True) self.assertTrue(search_results.total == 1) # some more tx_state checks before moving on to the next thing tx_state = self._get_transaction_state() self.assertTrue(len(tx_state.pending_updates) == 0) self.assertTrue(len(tx_state.indexed_updates) == 2) self.assertTrue(len(tx_state.temp_indexed_uids) == 1) self.assertTrue(len(tx_state.temp_deleted_uids) == 1) # Simulate transaction is committed and do checks updated_uids = set( tx_state.pending_updates.keys()) | tx_state.temp_indexed_uids try: tid = self.data_manager._get_tid() # before commit we should have 2 docs with tx_state = tid mi_results = self.model_index.search( SearchParams(Eq(TX_STATE_FIELD, tid))) self.assertTrue(mi_results.total_count == 2) # Lets do the commit self._simulate_tx_commit() self.assertIsNone(self._get_transaction_state()) # Check we only have one doc matching "/zport/dmd/Devices/device_class*" search_results = self.model_catalog.search(query=query, commit_dirty=False) self.assertEquals(search_results.total, 1) # Check the result's tx_state field has been set to zero brain = search_results.results.next() self.assertEquals(brain.tx_state, 0) # No documents should remain with tx_state == tid mi_results = self.model_index.search( SearchParams(Eq(TX_STATE_FIELD, tid))) self.assertEquals(mi_results.total_count, 0) finally: # clean up created docs in solr query = In(UID, updated_uids) self.model_index.unindex_search(SearchParams(query)) # create another organizer in a new transaction dc_3 = self.dmd.Devices.createOrganizer(device_class_3) dc_3_uid = dc_3.idx_uid() self._check_tx_state(pending=dc_3_uid) tx_state = self._get_transaction_state() self.assertTrue(len(tx_state.pending_updates) == 1) self.assertTrue(len(tx_state.indexed_updates) == 0) self.assertTrue(len(tx_state.temp_indexed_uids) == 0) self.assertTrue(len(tx_state.temp_deleted_uids) == 0) # Manual mid-transaction commit self.data_manager.do_mid_transaction_commit() self._check_tx_state(temp_indexed=dc_3_uid) self.assertTrue(len(tx_state.pending_updates) == 0) self.assertTrue(len(tx_state.indexed_updates) == 1) self.assertTrue(len(tx_state.temp_indexed_uids) == 1) self.assertTrue(len(tx_state.temp_deleted_uids) == 0) query = MatchGlob(UID, "/zport/dmd/Devices/device_class*") search_results = self.model_catalog.search(query=query, commit_dirty=False) self._validate_temp_indexed_results(search_results, expected_object_uids=[dc_3_uid]) # Simulate transaction is aborted and check tx state has been reset self.data_manager.abort(transaction.get()) # No docs should match the device class uid search_results = self.model_catalog.search(query=Eq(UID, dc_3_uid), commit_dirty=False) self.assertTrue(search_results.total == 0) # No documents should remain with tx_state == tid tid = self.data_manager._get_tid() mi_results = self.model_index.search( SearchParams(Eq(TX_STATE_FIELD, tid))) self.assertEquals(mi_results.total_count, 0) self.assertIsNone(self._get_transaction_state()) # delete a doc that exists before current tx, do a search with commit dirty and abort dc_4 = self.dmd.Devices.createOrganizer(device_class_4) dc_4_uid = dc_4.idx_uid() query = Eq(UID, dc_4_uid) try: self._simulate_tx_commit( ) # commit to get the device_class_4 doc in solr # check the doc exists in solr search_results = self.model_catalog.search(query=query) self.assertTrue(search_results.total == 1) # delete the object self.dmd.Devices._delObject(device_class_4) # a model catalog search with commit_dirty=True should no return the deleted doc search_results = self.model_catalog.search(query=query, commit_dirty=True) self.assertTrue(search_results.total == 0) # however the doc is still in solr mi_results = self.model_index.search(SearchParams(query)) self.assertTrue(mi_results.total_count == 1) # Abort tx self.data_manager.abort(transaction.get()) # The doc should have been left intact in solr search_results = self.model_catalog.search(query=query) self.assertTrue(search_results.total == 1) finally: # clean up created docs in solr self.model_index.unindex_search(SearchParams(query))
def listMatchGlob(op, index, list): return op(*[MatchGlob(index, '*%s*' % i) for i in list])
def _parse_basic_query(attr, value): if isinstance(value, str) and '*' in value: return MatchGlob(attr, value) else: return Eq(attr, value)
def find_claimable_device(self, device_class=None): ''' Find a possible Linux device for the host: Search by id, title, and management IP, against id, hostnames, and IPs ''' if device_class is None: device_class = self.proxy_deviceclass() suggested_name = self.suggested_device_name() search_values = [ x for x in self.id, suggested_name, self.hostname, self.host_ip if x is not None ] brains = device_class.deviceSearch.evalAdvancedQuery( And( MatchGlob('getDeviceClassPath', device_class.getOrganizerName() + "*"), Or(In('id', search_values), In('titleOrId', search_values), In('getDeviceIp', search_values)))) possible_devices = [] for brain in brains: try: device = brain.getObject() if device.openstack_hostComponent() is None: if hasattr(device, 'getIpRealm'): if self.getIpRealm() is device.getIpRealm(): possible_devices.append(device) else: possible_devices.append(device) else: LOG.info( "%s component %s unable to claim device %s, because it is already linked to %s", self.meta_type, self.name(), device.id, device.openstack_hostComponent().id) except Exception: pass # 1. First look by matching id against my id/suggested_name/hostname for device in possible_devices: if device.id == self.id: return device for device in possible_devices: if device.id == suggested_name or device.id == self.hostname: return device # 2. Next find by matching name against my id/suggested_name/hostname for device in possible_devices: if device.name() == self.id: return device for device in possible_devices: if device.name() == suggested_name or device.name( ) == self.hostname: return device # Otherwise, return the first device, if one was found if possible_devices: return possible_devices[0] if device_class == self.proxy_deviceclass(): # check for other devices that we would have claimed, if they # had been in the right device class device = self.find_claimable_device(device_class=self.dmd.Devices) if device: LOG.info( "No claimable device found for %s, but %s was found " "in another device class. Moving it to %s will make " "it eligible.", self.id, device.id, self.proxy_deviceclass().getOrganizerName()) # No claimable device was found. return None