def _getSearchResultsFromModelCatalog(self,
parsedQuery,
sorter=None,
category=None,
countOnly=False,
unrestricted=False,
filterFn=None,
maxResults=None):
operators = parsedQuery.operators
keywords = parsedQuery.keywords
if not keywords:
return
def listMatchGlob(op, index, list):
return op(*[MatchGlob(index, '*%s*' % i) for i in list])
dmd = self._dmd
kw_query = Or(listMatchGlob(And, 'name', keywords),
listMatchGlob(And, 'text_ipAddress', keywords))
# Rank devices whose name match the query higher than other stuff
# TODO: Figure out how to expose Lucene boosts
# For now we just or the boost query in with the original query to boost those results
ranker = listMatchGlob(Or, 'name', keywords)
full_query = Or(kw_query, ranker)
cat = IModelCatalogTool(dmd).devices
limit = 0 if countOnly and not filterFn else maxResults
# Set orderby to None so that modelindex will rank by score
catalogItems = cat.search(query=full_query,
orderby=None,
filterPermissions=True,
limit=limit)
brainResults = [
DeviceSearchResult(catalogItem) for catalogItem in catalogItems
]
if countOnly and not filterFn:
return dict(Device=brainResults.total)
if filterFn:
brainResults = filter(filterFn, brainResults)
if countOnly:
return dict(Device=len(brainResults))
results = brainResults
if sorter is not None:
results = sorter.limitSort(results)
return results
def getObjectBrains(self,
uid=None,
start=0,
limit=50,
sort='name',
dir='ASC',
params=None,
hashcheck=None,
types=(),
fields=[]):
cat = IModelCatalogTool(self._getObject(uid))
reverse = bool(dir == 'DESC')
qs = []
query = None
globFilters = {}
prodStates = None
params = params if params else {}
for key, value in params.iteritems():
if key == 'ipAddress':
qs.append(MatchGlob('text_ipAddress', '{}*'.format(value)))
elif key == 'productionState':
qs.append(
Or(*[Eq('productionState', str(state))
for state in value]))
# ZEN-30949 - stringify values from the 'priority' list if it's passed in for query criteria
elif key == 'priority':
qs.append(
Or(*[Eq('priority', str(priority)) for priority in value]))
# ZEN-10057 - move filtering on indexed groups/systems/location from post-filter to query
elif key in organizersToClass:
organizerQuery = self.findMatchingOrganizers(
organizersToClass[key], organizersToPath[key], value)
if not organizerQuery:
return []
qs.append(organizerQuery)
else:
globFilters[key] = value
if qs:
query = And(*qs)
return cat.search(types,
start=start,
limit=limit,
orderby=sort,
reverse=reverse,
query=query,
globFilters=globFilters,
hashcheck=hashcheck,
fields=fields)
def getDevProdStateJSON(self, prodStates=['Maintenance']):
"""
Return a map of device to production state in a format suitable for a
YUI data table.
@return: A JSON representation of a dictionary describing devices
@rtype: "{
'columns':['Device', 'Prod State'],
'data':[
{'Device':'<a href=/>', 'Prod State':'Production'},
{'Device':'<a href=/>', 'Prod State':'Maintenance'},
]}"
"""
devroot = self.context.dmd.Devices
if isinstance(prodStates, basestring):
prodStates = [prodStates]
orderby, orderdir = 'id', 'asc'
catalog = getattr(devroot, devroot.default_catalog)
queries = []
for state in prodStates:
queries.append(Eq('getProdState', state))
query = Or(*queries)
objects = catalog.evalAdvancedQuery(query, ((orderby, orderdir),))
devs = (x.getObject() for x in objects)
mydict = {'columns':['Device', 'Prod State'], 'data':[]}
for dev in devs:
if not self.context.checkRemotePerm(ZEN_VIEW, dev): continue
mydict['data'].append({
'Device' : dev.getPrettyLink(),
'Prod State' : dev.getProdState()
})
if len(mydict['data'])>=100:
break
return mydict
def getDeviceBrains(self, uid=None, start=0, limit=50, sort='name',
dir='ASC', params=None, hashcheck=None):
cat = ICatalogTool(self._getObject(uid))
reverse = dir=='DESC'
qs = []
query = None
globFilters = {}
if params is None:
params = {}
for key, value in params.iteritems():
if key == 'ipAddress':
ip = ensureIp(value)
try:
checkip(ip)
except IpAddressError:
pass
else:
if numbip(ip):
minip, maxip = getSubnetBounds(ip)
qs.append(Between('ipAddress', str(minip), str(maxip)))
elif key == 'deviceClass':
qs.append(MatchRegexp('uid', '(?i).*%s.*' %
value))
elif key == 'productionState':
qs.append(Or(*[Eq('productionState', str(state))
for state in value]))
else:
globFilters[key] = value
if qs:
query = And(*qs)
brains = cat.search('Products.ZenModel.Device.Device', start=start,
limit=limit, orderby=sort, reverse=reverse,
query=query, globFilters=globFilters, hashcheck=hashcheck)
return brains
def _getAdvancedQueryDeviceList(self,
offset=0,
count=50,
filter='',
orderby='name',
orderdir='asc'):
"""
Ask the catalog for devices matching the criteria specified.
"""
context = self.context
if not isinstance(context, DeviceOrganizer):
context = self.context.dmd.Devices
catalog = IModelCatalogTool(context).devices
devfilter = '(?is).*%s.*' % filter
filterquery = Or(MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter))
query = Eq('uid', context.absolute_url_path()) & filterquery
objects = catalog.search(query=query,
start=int(offset),
limit=int(count),
order_by=orderby,
reverse=orderdir != 'asc')
objects = list(objects)
totalCount = len(objects)
return totalCount, objects
def get_uids(index_client, root="", types=()):
start = 0
need_results = True
query = [Eq("tx_state", 0)]
if root:
root = root.rstrip('/')
query.append(
Or(Eq("uid", "{}".format(root)),
MatchGlob("uid", "{}/*".format(root))))
if not isinstance(types, (tuple, list)):
types = (types, )
if types:
query.append(In("objectImplements", [dottedname(t) for t in types]))
while need_results:
search_results = index_client.search(
SearchParams(query=And(*query),
start=start,
limit=MODEL_INDEX_BATCH_SIZE,
order_by="uid",
fields=["uid"]))
start += MODEL_INDEX_BATCH_SIZE
for result in search_results.results:
yield result.uid
need_results = start < search_results.total_count
def validate_global_catalog(self):
# Get Devices using AdvancedQuery
query = Eq("objectImplements", "Products.ZenModel.Device.Device")
brains = self.global_catalog.search(query)
self.assertEqual(len(brains), self.n_devices)
# Get Devices using dict query
query = {"objectImplements": "Products.ZenModel.Device.Device"}
brains = self.global_catalog.search(query)
self.assertEqual(len(brains), self.n_devices)
# Get Locations
query = Eq("objectImplements", "Products.ZenModel.Location.Location")
brains = self.global_catalog.search(query)
self.assertEqual(len(brains),
self.n_devices + 1) # n_devices plus root node
# Get Devices
query = []
query.append(Eq("objectImplements", "Products.ZenModel.Device.Device"))
query.append(
Eq("objectImplements", "Products.ZenModel.IpAddress.IpAddress"))
brains = self.global_catalog.search(Or(*query))
self.assertEqual(len(brains), 2 * self.n_devices)
# Get a device by uid
d = self.devices[0]
uid = "/".join(d.device.getPrimaryPath())
# global_catalog strips /zport/dmd/
uid = uid[10:]
# using AdvancedQuery
brains = self.global_catalog.search(Eq("uid", uid))
self.assertEqual(len(brains), 1)
# using dict query
brains = self.global_catalog.search({"uid": uid})
self.assertEqual(len(brains), 1)
def _buildQuery(self, types, paths, depth, query, filterPermissions):
qs = []
if query is not None:
qs.append(query)
# Build the path query
if not paths:
paths = ('/'.join(self.context.getPhysicalPath()), )
q = {'query': paths}
if depth is not None:
q['depth'] = depth
pathq = Generic('path', q)
qs.append(pathq)
# Build the type query
if not isinstance(types, (tuple, list)):
types = (types, )
subqs = [Eq('objectImplements', dottedname(t)) for t in types]
if subqs:
# Don't unnecessarily nest in an Or if there is only one type query
typeq = subqs[0] if len(subqs) == 1 else Or(*subqs)
qs.append(typeq)
# filter based on permissions
if filterPermissions:
qs.append(
In('allowedRolesAndUsers',
allowedRolesAndGroups(self.context)))
# Consolidate into one query
return And(*qs)
def _typecatComponentBrains(self,
uid=None,
types=(),
meta_type=(),
start=0,
limit=None,
sort='name',
dir='ASC',
name=None,
keys=()):
obj = self._getObject(uid)
spec = get_component_field_spec(meta_type)
if spec is None:
return None, 0
typecat = spec.get_catalog(obj, meta_type)
sortspec = ()
if sort:
if sort not in typecat._catalog.indexes:
# Fall back to slow queries and sorting
return None, 0
sortspec = ((sort, dir), )
querySet = [Generic('path', uid)]
if name:
querySet.append(
Or(*(MatchGlob(field, '*%s*' % name)
for field in spec.fields)))
brains = typecat.evalAdvancedQuery(And(*querySet), sortspec)
total = len(brains)
if limit is None:
brains = brains[start:]
else:
brains = brains[start:start + limit]
return brains, total
def searchForMembers(self, REQUEST=None, **kw):
"""Search for users (not groups, not groups-as-users) of a site """
if REQUEST:
dict = REQUEST
else:
dict = kw
name = dict.get('name', '')
# Split up search terms but leave quoted ones together
try:
names = shlex.split(name)
except ValueError:
try:
names = shlex.split(name.replace("'", "\\'"))
except ValueError:
names = shlex.split(name.replace("'", "\\'") + '"')
# Short circuit: if all that was asked for was '*', just return everyone
if names == ['*']:
query = And()
else:
queries = []
for name in names:
queries.extend([
MatchGlob('fullname', name),
MatchGlob('email', name),
Eq('getUserName', name)
])
query = Or(*queries)
zLOG.LOG('MembershipTool', zLOG.BLATHER, 'Querying: %s' % query)
catalog = getToolByName(self, 'member_catalog')
return catalog.evalAdvancedQuery(query, ('surname', 'firstname'))
def _getDeviceBatch(self,
selectstatus='none',
goodevids=[],
badevids=[],
offset=0,
count=50,
filter='',
orderby='titleOrId',
orderdir='asc'):
unused(count, offset, orderby, orderdir)
if not isinstance(goodevids, (list, tuple)):
goodevids = [goodevids]
if not isinstance(badevids, (list, tuple)):
badevids = [badevids]
if selectstatus == 'all':
idquery = ~In('id', badevids)
else:
idquery = In('id', goodevids)
devfilter = '(?is).*%s.*' % filter
filterquery = Or(MatchRegexp('id', devfilter),
MatchRegexp('name', devfilter),
MatchRegexp('text_ipAddress', devfilter),
MatchRegexp('deviceClassPath', devfilter))
query = Eq('uid', self.context.absolute_url_path()) & idquery
query = query & filterquery
catalog = IModelCatalogTool(self.context)
objects = catalog.search(query=query)
return [x['id'] for x in objects]
def _findDevices(self, identifier, ipAddress, limit=None):
"""
Returns a tuple ([device brains], [devices]) searching manage IP and
interface IPs. limit is the maximum total number in both lists.
"""
dev_cat = IModelCatalogTool(self._devices)
try:
ip_address = next(i for i in (ipAddress, identifier) if isip(i))
ip_decimal = ipToDecimal(ip_address)
except Exception:
ip_address = None
ip_decimal = None
quoted_id = quote_and_escape(identifier)
query_set = Or(Eq('id', quoted_id), Eq('name', quoted_id))
if ip_decimal is not None:
query_set.addSubquery(Eq('decimal_ipAddress', str(ip_decimal)))
device_brains = list(
dev_cat.search(types=Device,
query=query_set,
limit=limit,
filterPermissions=False,
fields=["uid", "uuid"]))
if device_brains:
return device_brains, []
if ip_decimal is not None:
# don't search interfaces for 127.x.x.x IPv4 addresses
if ipToDecimal('126.255.255.255') < ip_decimal < ipToDecimal(
'128.0.0.0'):
ip_decimal = None
# don't search interfaces for the ::1 IPv6 address
elif ipToDecimal('::1') == ip_decimal:
ip_decimal = None
if ip_decimal is None:
return [], []
net_cat = IModelCatalogTool(self._networks)
results = net_cat.search(types=IpAddress,
query=(Eq('name', ip_address)),
limit=limit,
filterPermissions=False)
devices = [brain.getObject().device() for brain in results]
return device_brains, devices
def _findDevice(self, devicename, useTitle=True):
"""
Returns all devices whose ip/id/title match devicename.
ip/id matches are at the front of the list.
@rtype: list of brains
"""
idIpQuery = Or(MatchGlob('id', devicename),
Eq('getDeviceIp', devicename))
if useTitle:
titleOrIdQuery = MatchGlob('titleOrId', devicename)
query = Or(idIpQuery, titleOrIdQuery)
rankSort = RankByQueries_Max((idIpQuery, 16), (titleOrIdQuery, 8))
devices = self._getCatalog().evalAdvancedQuery(query, (rankSort, ))
else:
devices = self._getCatalog().evalAdvancedQuery(idIpQuery)
return devices
def _findDevices(self, identifier, ipAddress, limit=None):
"""
Returns a tuple ([device brains], [devices]) searching manage IP and
interface IPs. limit is the maximum total number in both lists.
"""
dev_cat = IModelCatalogTool(self._devices)
try:
ip_address = next(i for i in (ipAddress, identifier) if isip(i))
ip_decimal = ipToDecimal(ip_address)
except Exception:
ip_address = None
ip_decimal = None
quoted_id = quote_and_escape(identifier)
query_set = Or(Eq('id', quoted_id), Eq('name', quoted_id))
if ip_decimal is not None:
query_set.addSubquery(Eq('decimal_ipAddress', str(ip_decimal)))
device_brains = list(dev_cat.search(types=Device,
query=query_set,
limit=limit,
filterPermissions=False,
fields=["uid", "uuid"]))
if device_brains:
return device_brains, []
if ip_decimal is not None:
# don't search interfaces for 127.x.x.x IPv4 addresses
if ipToDecimal('126.255.255.255') < ip_decimal < ipToDecimal('128.0.0.0'):
ip_decimal = None
# don't search interfaces for the ::1 IPv6 address
elif ipToDecimal('::1') == ip_decimal:
ip_decimal = None
if ip_decimal is None:
return [], []
net_cat = IModelCatalogTool(self._networks)
results = net_cat.search(types=IpAddress,
query=(Eq('name', ip_address)),
limit = limit,
filterPermissions = False)
devices = [brain.getObject().device() for brain in results]
return device_brains, devices
def devices(self, name):
"""return managed devices from ip, name or mac address
taken from ZentinelPortal.py"""
zcatalog = self.dmd.Devices.deviceSearch
glob = name.rstrip('*') + '*'
glob = MatchGlob('id', glob)
query = Or(glob, Eq('getDeviceIp', name))
brains = zcatalog.evalAdvancedQuery(query)
brains += self.dmd.Networks.ipSearch.evalAdvancedQuery(glob)
return [b.getObject() for b in brains]
def set_hostByName(self, name):
if not name:
log.warning("Could not set host. Given name is None")
return
query = Or(Eq('hostname', name), Eq('hostfqdn', name))
hosts = self.search('Host', query)
if len(hosts) > 0:
if len(hosts) > 1:
log.warning("Got more than one host for hypervisor: " +
"%s with id: %s" % (self.title, self.id))
try:
host = hosts[0].getObject()
except Exception:
# ignore a stale entry
pass
else:
log.info("Set host by fqdn: %s" % name)
self.set_host(host.id)
self.hostfqdn = name
elif name.find('.') > -1:
name = name[:name.index('.')]
query = Or(Eq('hostname', name), Eq('hostfqdn', name))
hosts = self.search('Host', query)
if len(hosts) > 0:
if len(hosts) > 1:
log.warning("Got more than one host for hypervisor: " +
"%s with id: %s" (self.title, self.id))
try:
host = hosts[0].getObject()
except Exception:
# ignore a stale entry
pass
else:
log.info("Set host by hostname: %s" % name)
self.set_host(host.id)
self.hostfqdn = name
else:
log.error("%s: Could not set hypervisor host by name (%s)",
self.id, name)
def _componentSearch(self, uid=None, types=(), meta_type=(), start=0,
limit=None, sort='name', dir='ASC', name=None, keys=()):
reverse = dir=='DESC'
if isinstance(types, basestring):
types = (types,)
if isinstance(meta_type, basestring):
meta_type = (meta_type,)
querySet = []
if meta_type:
querySet.append(Or(*(Eq('meta_type', t) for t in meta_type)))
querySet.append(Generic('getAllPaths', uid))
query = And(*querySet)
obj = self._getObject(uid)
cat = obj.device().componentSearch
if 'getAllPaths' not in cat.indexes():
obj.device()._createComponentSearchPathIndex()
brains = cat.evalAdvancedQuery(query)
# unbrain the results
comps=map(IInfo, map(unbrain, brains))
# filter the components
if name is not None:
comps = self._filterComponents(comps, keys, name)
total = len(comps)
hash_ = str(total)
def componentSortKey(parent):
val = getattr(parent, sort)
if val:
if isinstance(val, list):
val = val[0]
if callable(val):
val = val()
if IInfo.providedBy(val):
val = val.name
return val
# sort the components
sortedResults = list(sorted(comps, key=componentSortKey, reverse=reverse))
# limit the search results to the specified range
if limit is None:
pagedResult = sortedResults[start:]
else:
pagedResult = sortedResults[start:start + limit]
# fetch any rrd data necessary
self.bulkLoadMetricData(pagedResult)
return SearchResults(iter(pagedResult), total, hash_, False)
def searchDevices(self, queryString='', REQUEST=None):
"""Returns the concatenation of a device name, ip and mac
search on the list of devices.
"""
# TODO: Remove. Not used anymore in Zenoss code --Ian
zcatalog = self.dmd.Devices.deviceSearch
glob = queryString.rstrip('*') + '*'
idGlob = MatchGlob('id', glob)
titleGlob = MatchGlob('titleOrId', glob)
idOrTitleQuery = Or(idGlob,titleGlob)
query = Or(idOrTitleQuery, Eq('getDeviceIp', queryString))
additionalQuery = self._additionalQuery()
if additionalQuery:
query = And( query, additionalQuery )
brains = zcatalog.evalAdvancedQuery(query)
if REQUEST and len(brains) == 1:
raise Redirect(urllib.quote(brains[0].getPrimaryId))
if additionalQuery:
idGlob = And( idGlob, additionalQuery )
brains += self.dmd.Networks.ipSearch.evalAdvancedQuery(idGlob)
return [ b.getObject() for b in brains ]
def storage_disk_lun(self):
# return the UCS storage disk/virtual drive
# if disk_ids contains the IDs of disk/virtual drive
if self.disk_ids:
results = ICatalogTool(self.getDmdRoot('Devices')).search(query=Or(
*[Eq('searchKeywords', id) for id in self.disk_ids]))
for brain in results:
try:
yield brain.getObject()
except Exception:
continue
def _componentSearch(self,
uid=None,
types=(),
meta_type=(),
start=0,
limit=None,
sort='name',
dir='ASC',
name=None,
keys=()):
reverse = dir == 'DESC'
if isinstance(types, basestring):
types = (types, )
if isinstance(meta_type, basestring):
meta_type = (meta_type, )
querySet = []
if meta_type:
querySet.append(Or(*(Eq('meta_type', t) for t in meta_type)))
querySet.append(Generic('getAllPaths', uid))
query = And(*querySet)
obj = self._getObject(uid)
if getattr(aq_base(obj.device()), 'componentSearch', None) is None:
obj.device()._create_componentSearch()
cat = obj.device().componentSearch
if 'getAllPaths' not in cat.indexes():
obj.device()._createComponentSearchPathIndex()
brains = cat.evalAdvancedQuery(query)
# unbrain the results
comps = map(IInfo, map(unbrain, brains))
total = len(comps)
hash_ = str(total)
# filter the components
if name is not None:
wrapped = map(IInfo, map(unbrain, brains))
comps = self._filterComponents(wrapped, keys, name)
total = len(comps)
hash_ = str(total)
# sort the components
sortedResults = list(
sorted(comps, key=lambda x: getattr(x, sort), reverse=reverse))
# limit the search results to the specified range
if limit is None:
pagedResult = sortedResults[start:]
else:
pagedResult = sortedResults[start:start + limit]
return SearchResults(iter(pagedResult), total, hash_, False)
def getSearchResults(self,
parsedQuery,
sorter=None,
category=None,
countOnly=False,
unrestricted=False,
filterFn=None):
"""
Queries the catalog. Searches the searchKeywords index
using *keyword1* AND *keyword2* AND so on.
If there are preferred categories, find maxResults # of instances
before searching other categories.
@rtype generator of BrainSearchResult objects
"""
operators = parsedQuery.operators
keywords = parsedQuery.keywords
if not keywords:
return
def listMatchGlob(op, index, list):
return op(*[MatchGlob(index, '*%s*' % i) for i in list])
dmd = self._dmd
kw_query = Or(listMatchGlob(And, 'titleOrId', keywords),
listMatchGlob(And, 'getDeviceIp', keywords))
# Rank devices whose name match the query higher than other stuff
ranker = RankByQueries_Sum(
(listMatchGlob(Or, 'titleOrId', keywords), 10), )
full_query = kw_query
cat = dmd.Devices.deviceSearch
querySet = full_query
catalogItems = cat.evalAdvancedQuery(querySet, (ranker, ))
brainResults = [
DeviceSearchResult(catalogItem) for catalogItem in catalogItems
if checkPermission("View", catalogItem.getObject())
]
if filterFn:
brainResults = filter(filterFn, brainResults)
if countOnly:
return dict(Device=len(brainResults))
results = brainResults
if sorter is not None:
results = sorter.limitSort(results)
return results
def _add_tx_state_query(self, search_params, tx_state):
"""
only interested in docs indexed by committed transactions or
in docs temporary committed by the current transaction
"""
values = [0] # default tx_state for committed transactions
if tx_state:
values.append(tx_state.tid)
if isinstance(search_params.query, dict):
search_params.query[TX_STATE_FIELD] = values
else: # We assume it is an AdvancedQuery
or_query = [Eq(TX_STATE_FIELD, value) for value in values]
search_params.query = And(search_params.query, Or(*or_query))
return search_params
def _getSearchResultsFromDeviceSearchCatalog(self,
parsedQuery,
sorter=None,
category=None,
countOnly=False,
unrestricted=False,
filterFn=None,
maxResults=None):
"""
DEPRECATED. the DeviceSearch catalog will be replaced by the Model Catalog
"""
operators = parsedQuery.operators
keywords = parsedQuery.keywords
if not keywords:
return
def listMatchGlob(op, index, list):
return op(*[MatchGlob(index, '*%s*' % i) for i in list])
dmd = self._dmd
kw_query = Or(listMatchGlob(And, 'titleOrId', keywords),
listMatchGlob(And, 'getDeviceIp', keywords))
# Rank devices whose name match the query higher than other stuff
ranker = RankByQueries_Sum(
(listMatchGlob(Or, 'titleOrId', keywords), 10), )
full_query = kw_query
cat = dmd.Devices.deviceSearch
querySet = full_query
catalogItems = cat.evalAdvancedQuery(querySet, (ranker, ))
brainResults = [
DeviceSearchResult(catalogItem) for catalogItem in catalogItems
if checkPermission("View", catalogItem.getObject())
]
if filterFn:
brainResults = filter(filterFn, brainResults)
if countOnly:
return dict(Device=len(brainResults))
results = brainResults
if sorter is not None:
results = sorter.limitSort(results)
return results
def _findDevice(self, devicename, useTitle=True, commit_dirty=False):
"""
Returns all devices whose ip/id/title match devicename.
ip/id matches are at the front of the list.
@rtype: list of brains
"""
ors = [ MatchGlob('id', devicename), MatchGlob('text_ipAddress', devicename) ]
if useTitle:
ors.append(MatchGlob('name', devicename))
query = And( Eq("objectImplements", "Products.ZenModel.Device.Device"), Or(*ors) )
fields = [ "name", "id", "text_ipAddress" ]
search_results = IModelCatalogTool(self.dmd.Devices).search(query=query, fields=fields, commit_dirty=commit_dirty)
return list(search_results.results)
def get_possible_faqs(self):
log.info('get_possible_faqs')
queries = []
title = In('Title', ["*frequently*", "*faq*", "FAQ*", "Frequently*"])
portal_type = In("portal_type", ["Document", "RichDocument", "Folder"])
ids = ["faq", "faq.php", "faq.stm", "faqs"]
for i in range(0, 10):
ids.append('faq%d.stm' % i)
ids.append('faq0%d.php' % i)
id = In('getId', ids)
body = Eq('SearchableText', "FAQ")
fop = Eq('path', '/osha/portal/fop')
advanced_query = And(Or(id, title, body), portal_type, Not(fop))
ls = self.portal_catalog.evalAdvancedQuery(advanced_query,
(('Date', 'desc'), ))
# XXX: Didn't work :(
# ls = self.portal_catalog(
# getId='faq.php',
# path='/osha/portal/en/good_practice/priority_groups/disability/')
ls = self.portal_catalog(
getId='faq2.stm',
path='osha/en/good_practice/topics/dangerous_substances/faq2.stm')
# ls = self.portal_catalog(
# getId='faq.php',
# path='osha/en/good_practice/topics/accident_prevention/')
log.info("Processing FAQs: %s" % "\n".join([i.getURL() for i in ls]))
odict = {}
for l in ls:
o = l.getObject()
odict[o.absolute_url()] = o
ts = o.getTranslations().values()
for t in ts:
odict[t[0].absolute_url()] = t[0]
objects = odict.values()
return objects
k = ['/'.join(o.getPhysicalPath()) for o in objects]
k.sort()
display_str = '\n'.join(k) or 'none'
return display_str
def setIdsInRelationship(self, relationship, ids):
"""Update ToMany relationship given relationship and ids."""
new_ids = set(ids)
current_ids = set(o.id for o in relationship.objectValuesGen())
changed_ids = new_ids.symmetric_difference(current_ids)
query = Or(*[Eq('id', x) for x in changed_ids])
obj_map = {}
for result in catalog_search(self.device(), 'ComponentBase', query):
try:
component = result.getObject()
except Exception as e:
self.LOG.error(
"Trying to access non-existent object {}".format(e))
else:
obj_map[result.id] = component
for id_ in new_ids.symmetric_difference(current_ids):
obj = obj_map.get(id_)
if not obj:
self.LOG.error(
"setIdsInRelationship ({}): No targets found matching "
"id={}".format(relationship, id_))
continue
if id_ in new_ids:
self.LOG.debug("Adding {} to {}".format(obj, relationship))
relationship.addRelation(obj)
# Index remote object. It might have a custom path reporter.
notify(IndexingEvent(obj, 'path', False))
else:
self.LOG.debug("Removing {} from {}".format(obj, relationship))
relationship.removeRelation(obj)
# If the object was not deleted altogether..
if not isinstance(relationship, ToManyContRelationship):
# Index remote object. It might have a custom path reporter.
notify(IndexingEvent(obj, 'path', False))
# For componentSearch. Would be nice if we could target
# idxs=['getAllPaths'], but there's a chance that it won't exist
# yet.
obj.index_object()
def updateToMany(relationship, root, type_, ids):
'''
Update ToMany relationship given search root, type and ids.
This is a general-purpose function for efficiently building
non-containing ToMany relationships.
'''
root = root.primaryAq()
new_ids = set(map(prepId, ids))
current_ids = set(o.id for o in relationship.objectValuesGen())
changed_ids = new_ids.symmetric_difference(current_ids)
query = Or(*(Eq('id', x) for x in changed_ids))
obj_map = {}
for result in ICatalogTool(root).search(types=[type_], query=query):
obj_map[result.id] = result.getObject()
for id_ in new_ids.symmetric_difference(current_ids):
obj = obj_map.get(id_)
if not obj:
continue
if id_ in new_ids:
relationship.addRelation(obj)
# Index remote object. It might have a custom path reporter.
notify(IndexingEvent(obj, 'path', False))
else:
relationship.removeRelation(obj)
# If the object was not deleted altogether..
if not isinstance(relationship, ToManyContRelationship):
# Index remote object. It might have a custom path reporter.
notify(IndexingEvent(obj, 'path', False))
# For componentSearch. Would be nice if we could target
# Index remote object. It might have a custom path reporter.
notify(IndexingEvent(obj, 'path', False))
# For componentSearch. Would be nice if we could target
# idxs=['getAllPaths'], but there's a chance that it won't exist
# yet.
obj.index_object()
def getElementUuidById(self, catalog, element_type_id, id):
"""
Find element by ID but only cache UUID. This forces us to lookup elements
each time by UUID (pretty fast) which gives us a chance to see if the element
has been deleted.
"""
cls = self.ELEMENT_TYPE_MAP.get(element_type_id)
if cls:
catalog = catalog or self._catalogs.get(element_type_id)
if catalog:
results = ICatalogTool(catalog).search(cls,
query=Or(Eq('id', id),
Eq('name', id)),
filterPermissions=False,
limit=1)
if results.total:
return self.uuidFromBrain(results.results.next())
def getDeviceBrains(self, uid=None, start=0, limit=50, sort='name',
dir='ASC', params=None, hashcheck=None):
cat = ICatalogTool(self._getObject(uid))
reverse = bool(dir == 'DESC')
qs = []
query = None
globFilters = {}
if params is None:
params = {}
for key, value in params.iteritems():
if key == 'ipAddress':
ip = ensureIp(value)
try:
checkip(ip)
except IpAddressError:
pass
else:
if numbip(ip):
minip, maxip = getSubnetBounds(ip)
qs.append(Between('ipAddress', str(minip), str(maxip)))
# ZEN-10057 - move filtering on indexed groups/systems/location from post-filter to query
elif key in organizersToClass:
organizerQuery = self.findMatchingOrganizers(organizersToClass[key], organizersToPath[key], value)
if not organizerQuery:
return []
qs.append(organizerQuery)
elif key == 'productionState':
qs.append(Or(*[Eq('productionState', str(state))
for state in value]))
else:
globFilters[key] = value
if qs:
query = And(*qs)
brains = cat.search(
'Products.ZenModel.Device.Device', start=start,
limit=limit, orderby=sort, reverse=reverse,
query=query, globFilters=globFilters, hashcheck=hashcheck
)
return brains
def transform_query(self, query):
search_indexes = []
other_indexes = []
search_keys = ('customer', 'Subject', 'No matches by customer')
for key in query.keys():
if key in ('b_start', 'b_size', 'batch', 'show_inactive'):
continue
if key in search_keys:
if key == 'customer' and query['customer'] == 'All':
for customer in self.customers_list():
if customer in ('All', 'No matches by customer'):
continue
search_indexes.append(Eq('customer', customer))
elif (key == 'customer' and
query['customer'] == 'No matches by customer'):
pass
else:
self.add_to_index_list(search_indexes, query, key)
else:
self.add_to_index_list(other_indexes, query, key)
other_indexes = And(*other_indexes)
search_indexes = Or(*search_indexes)
return And(other_indexes, search_indexes)
def getElementUuidById(self, catalog, element_type_id, id):
"""
Find element by ID but only cache UUID. This forces us to lookup elements
each time by UUID (pretty fast) which gives us a chance to see if the element
has been deleted.
"""
cls = self.ELEMENT_TYPE_MAP.get(element_type_id)
if cls:
catalog = catalog or self._catalogs.get(element_type_id)
if catalog:
quoted_id = quote_and_escape(id)
results = IModelCatalogTool(catalog).search(
cls,
query=Or(Eq('id', quoted_id), Eq('name', quoted_id)),
filterPermissions=False,
limit=1,
fields=["uuid"])
if results.total:
try:
result = results.results.next()
except StopIteration:
pass
else:
return self.uuidFromBrain(result)
def folderitems(self, full_objects = False):
"""
>>> portal = layer['portal']
>>> portal_url = portal.absolute_url()
>>> from plone.app.testing import SITE_OWNER_NAME
>>> from plone.app.testing import SITE_OWNER_PASSWORD
Test page batching https://github.com/bikalabs/Bika-LIMS/issues/1276
When visiting the second page, the Water sampletype should be displayed:
>>> browser = layer['getBrowser'](portal, loggedIn=True, username=SITE_OWNER_NAME, password=SITE_OWNER_PASSWORD)
>>> browser.open(portal_url+"/bika_setup/bika_sampletypes/folder_view?",
... "list_pagesize=10&list_review_state=default&list_pagenumber=2")
>>> browser.contents
'...Water...'
"""
#self.contentsMethod = self.context.getFolderContents
if not hasattr(self, 'contentsMethod'):
self.contentsMethod = getToolByName(self.context, self.catalog)
context = aq_inner(self.context)
plone_layout = getMultiAdapter((context, self.request), name = u'plone_layout')
plone_utils = getToolByName(context, 'plone_utils')
plone_view = getMultiAdapter((context, self.request), name = u'plone')
portal_properties = getToolByName(context, 'portal_properties')
portal_types = getToolByName(context, 'portal_types')
workflow = getToolByName(context, 'portal_workflow')
site_properties = portal_properties.site_properties
norm = getUtility(IIDNormalizer).normalize
if self.request.get('show_all', '').lower() == 'true' \
or self.show_all == True \
or self.pagesize == 0:
show_all = True
else:
show_all = False
pagenumber = int(self.request.get('pagenumber', 1) or 1)
pagesize = self.pagesize
start = (pagenumber - 1) * pagesize
end = start + pagesize - 1
if (hasattr(self, 'And') and self.And) \
or (hasattr(self, 'Or') and self.Or):
# if contentsMethod is capable, we do an AdvancedQuery.
if hasattr(self.contentsMethod, 'makeAdvancedQuery'):
aq = self.contentsMethod.makeAdvancedQuery(self.contentFilter)
if hasattr(self, 'And') and self.And:
tmpAnd = And()
for q in self.And:
tmpAnd.addSubquery(q)
aq &= tmpAnd
if hasattr(self, 'Or') and self.Or:
tmpOr = Or()
for q in self.Or:
tmpOr.addSubquery(q)
aq &= tmpOr
brains = self.contentsMethod.evalAdvancedQuery(aq)
else:
# otherwise, self.contentsMethod must handle contentFilter
brains = self.contentsMethod(self.contentFilter)
else:
brains = self.contentsMethod(self.contentFilter)
results = []
self.page_start_index = 0
current_index = -1
for i, obj in enumerate(brains):
# we don't know yet if it's a brain or an object
path = hasattr(obj, 'getPath') and obj.getPath() or \
"/".join(obj.getPhysicalPath())
if hasattr(obj, 'getObject'):
obj = obj.getObject()
# check if the item must be rendered or not (prevents from
# doing it later in folderitems) and dealing with paging
if not self.isItemAllowed(obj):
continue
# avoid creating unnecessary info for items outside the current
# batch; only the path is needed for the "select all" case...
# we only take allowed items into account
current_index += 1
if not show_all and not (start <= current_index <= end):
results.append(dict(path = path, uid = obj.UID()))
continue
uid = obj.UID()
title = obj.Title()
description = obj.Description()
icon = plone_layout.getIcon(obj)
url = obj.absolute_url()
relative_url = obj.absolute_url(relative = True)
fti = portal_types.get(obj.portal_type)
if fti is not None:
type_title_msgid = fti.Title()
else:
type_title_msgid = obj.portal_type
url_href_title = '%s at %s: %s' % (
t(type_title_msgid),
path,
to_utf8(description))
modified = self.ulocalized_time(obj.modified()),
# element css classes
type_class = 'contenttype-' + \
plone_utils.normalizeString(obj.portal_type)
state_class = ''
states = {}
for w in workflow.getWorkflowsFor(obj):
state = w._getWorkflowStateOf(obj).id
states[w.state_var] = state
state_class += "state-%s " % state
results_dict = dict(
obj = obj,
id = obj.getId(),
title = title,
uid = uid,
path = path,
url = url,
fti = fti,
item_data = json.dumps([]),
url_href_title = url_href_title,
obj_type = obj.Type,
size = obj.getObjSize,
modified = modified,
icon = icon.html_tag(),
type_class = type_class,
# a list of lookups for single-value-select fields
choices = {},
state_class = state_class,
relative_url = relative_url,
view_url = url,
table_row_class = "",
category = 'None',
# a list of names of fields that may be edited on this item
allow_edit = [],
# a list of names of fields that are compulsory (if editable)
required = [],
# "before", "after" and replace: dictionary (key is column ID)
# A snippet of HTML which will be rendered
# before/after/instead of the table cell content.
before = {}, # { before : "<a href=..>" }
after = {},
replace = {},
)
try:
self.review_state = workflow.getInfoFor(obj, 'review_state')
state_title = workflow.getTitleForStateOnType(
self.review_state, obj.portal_type)
state_title = t(PMF(state_title))
except:
self.review_state = 'active'
state_title = None
if self.review_state:
results_dict['review_state'] = self.review_state
for state_var, state in states.items():
if not state_title:
state_title = workflow.getTitleForStateOnType(
state, obj.portal_type)
results_dict[state_var] = state
results_dict['state_title'] = state_title
# extra classes for individual fields on this item { field_id : "css classes" }
results_dict['class'] = {}
for name, adapter in getAdapters((obj, ), IFieldIcons):
auid = obj.UID() if hasattr(obj, 'UID') and callable(obj.UID) else None
if not auid:
continue
alerts = adapter()
# logger.info(str(alerts))
if alerts and auid in alerts:
if auid in self.field_icons:
self.field_icons[auid].extend(alerts[auid])
else:
self.field_icons[auid] = alerts[auid]
# Search for values for all columns in obj
for key in self.columns.keys():
if hasattr(obj, key):
# if the key is already in the results dict
# then we don't replace it's value
if results_dict.has_key(key):
continue
value = getattr(obj, key)
if callable(value):
value = value()
results_dict[key] = value
results.append(results_dict)
return results
def __call__(self, result=None, specification=None, **kwargs):
searchTerm = _c(self.request.get('searchTerm', '')).lower()
force_all = self.request.get('force_all', 'false')
searchFields = 'search_fields' in self.request \
and json.loads(_u(self.request.get('search_fields', '[]'))) \
or ('Title',)
# lookup objects from ZODB
catalog_name = _c(self.request.get('catalog_name', 'portal_catalog'))
catalog = getToolByName(self.context, catalog_name)
# N.B. We don't use json.loads to avoid unicode conversion, which will
# fail in the catalog search for some cases
# see: https://github.com/senaite/bika.lims/issues/443
base_query = ast.literal_eval(self.request['base_query'])
search_query = ast.literal_eval(self.request.get('search_query', "{}"))
# first with all queries
contentFilter = dict((k, v) for k, v in base_query.items())
contentFilter.update(search_query)
# Sorted by? (by default, Title)
sort_on = self.request.get('sidx', 'Title')
if sort_on == 'Title':
sort_on = 'sortable_title'
if sort_on:
# Check if is an index and if is sortable. Otherwise, assume the
# sorting must be done manually
index = catalog.Indexes.get(sort_on, None)
if index and index.meta_type in ['FieldIndex', 'DateIndex']:
contentFilter['sort_on'] = sort_on
# Sort order?
sort_order = self.request.get('sord', 'asc')
if (sort_order in ['desc', 'reverse', 'rev', 'descending']):
contentFilter['sort_order'] = 'descending'
else:
contentFilter['sort_order'] = 'ascending'
# Can do a search for indexes?
criterias = []
fields_wo_index = []
if searchTerm:
for field_name in searchFields:
index = catalog.Indexes.get(field_name, None)
if not index:
fields_wo_index.append(field_name)
continue
if index.meta_type in ('ZCTextIndex'):
if searchTerm.isspace():
# earchTerm != ' ' added because of
# https://github.com/plone/Products.CMFPlone/issues
# /1537
searchTerm = ''
continue
else:
temp_st = searchTerm + '*'
criterias.append(MatchRegexp(field_name, temp_st))
elif index.meta_type in ('FieldIndex'):
criterias.append(MatchRegexp(field_name, searchTerm))
elif index.meta_type == 'DateIndex':
msg = "Unhandled DateIndex search on '%s'" % field_name
from bika.lims import logger
logger.warn(msg)
else:
criterias.append(Generic(field_name, searchTerm))
if criterias:
# Advanced search
advanced_query = catalog.makeAdvancedQuery(contentFilter)
aq_or = Or()
for criteria in criterias:
aq_or.addSubquery(criteria)
advanced_query &= aq_or
brains = catalog.evalAdvancedQuery(advanced_query)
else:
brains = catalog(contentFilter)
if brains and searchTerm and fields_wo_index:
_brains = []
for brain in brains:
for field_name in fields_wo_index:
value = getattr(brain, field_name, None)
if not value:
instance = brain.getObject()
schema = instance.Schema()
if field_name in schema:
value = schema[field_name].get(instance)
if callable(value):
value = value()
if value and value.lower().find(searchTerm) > -1:
_brains.append(brain)
break
brains = _brains
# Then just base_query alone ("show all if no match")
if not brains and force_all.lower() == 'true':
if search_query:
brains = catalog(base_query)
if brains and searchTerm:
_brains = [p for p in brains
if p.Title.lower().find(searchTerm) > -1]
if _brains:
brains = _brains
return brains
def folderitems(self, full_objects = False):
"""
"""
#self.contentsMethod = self.context.getFolderContents
if not hasattr(self, 'contentsMethod'):
self.contentsMethod = getToolByName(self.context, self.catalog)
context = aq_inner(self.context)
plone_layout = getMultiAdapter((context, self.request), name = u'plone_layout')
plone_utils = getToolByName(context, 'plone_utils')
plone_view = getMultiAdapter((context, self.request), name = u'plone')
portal_properties = getToolByName(context, 'portal_properties')
portal_types = getToolByName(context, 'portal_types')
workflow = getToolByName(context, 'portal_workflow')
site_properties = portal_properties.site_properties
norm = getUtility(IIDNormalizer).normalize
show_all = self.request.get('show_all', '').lower() == 'true'
pagenumber = int(self.request.get('pagenumber', 1) or 1)
pagesize = self.pagesize
start = (pagenumber - 1) * pagesize
end = start + pagesize
if (hasattr(self, 'And') and self.And) \
or (hasattr(self, 'Or') and self.Or):
# if contentsMethod is capable, we do an AdvancedQuery.
if hasattr(self.contentsMethod, 'makeAdvancedQuery'):
aq = self.contentsMethod.makeAdvancedQuery(self.contentFilter)
if hasattr(self, 'And') and self.And:
tmpAnd = And()
for q in self.And:
tmpAnd.addSubquery(q)
aq &= tmpAnd
if hasattr(self, 'Or') and self.Or:
tmpOr = Or()
for q in self.Or:
tmpOr.addSubquery(q)
aq &= tmpOr
brains = self.contentsMethod.evalAdvancedQuery(aq)
else:
# otherwise, self.contentsMethod must handle contentFilter
brains = self.contentsMethod(self.contentFilter)
else:
brains = self.contentsMethod(self.contentFilter)
results = []
self.page_start_index = ""
for i, obj in enumerate(brains):
# we don't know yet if it's a brain or an object
path = hasattr(obj, 'getPath') and obj.getPath() or \
"/".join(obj.getPhysicalPath())
# avoid creating unnecessary info for items outside the current
# batch; only the path is needed for the "select all" case...
if not show_all and not start <= i < end:
if hasattr(obj, 'getObject'):
uid = obj.UID
else:
uid = obj.UID()
results.append(dict(path = path, uid = uid))
continue
if self.page_start_index == "":
self.page_start_index = i
if hasattr(obj, 'getObject'):
obj = obj.getObject()
uid = obj.UID()
title = obj.Title()
description = obj.Description()
icon = plone_layout.getIcon(obj)
url = obj.absolute_url()
relative_url = obj.absolute_url(relative = True)
fti = portal_types.get(obj.portal_type)
if fti is not None:
type_title_msgid = fti.Title()
else:
type_title_msgid = obj.portal_type
url_href_title = u'%s at %s: %s' % \
(self.context.translate(type_title_msgid,
context = self.request),
path,
safe_unicode(description))
modified = self.ulocalized_time(obj.modified()),
# element css classes
type_class = 'contenttype-' + \
plone_utils.normalizeString(obj.portal_type)
state_class = ''
states = {}
for w in workflow.getWorkflowsFor(obj):
state = w._getWorkflowStateOf(obj).id
states[w.state_var] = state
state_class += "state-%s " % state
results_dict = dict(
obj = obj,
id = obj.getId,
title = title,
uid = uid,
path = path,
url = url,
fti = fti,
item_data = json.dumps([]),
url_href_title = url_href_title,
obj_type = obj.Type,
size = obj.getObjSize,
modified = modified,
icon = icon.html_tag(),
type_class = type_class,
# a list of lookups for single-value-select fields
choices = {},
state_class = state_class,
relative_url = relative_url,
view_url = url,
table_row_class = "",
# a list of names of fields that may be edited on this item
allow_edit = [],
# a list of names of fields that are compulsory (if editable)
required = [],
# "before", "after" and replace: dictionary (key is column ID)
# A snippet of HTML which will be rendered
# before/after/instead of the table cell content.
before = {}, # { before : "<a href=..>" }
after = {},
replace = {},
)
try:
review_state = workflow.getInfoFor(obj, 'review_state')
state_title = self.context.translate(
PMF(workflow.getTitleForStateOnType(review_state,
obj.portal_type)))
except:
review_state = 'active'
state_title = None
if review_state:
results_dict['review_state'] = review_state
for state_var, state in states.items():
if not state_title:
state_title = workflow.getTitleForStateOnType(
state, obj.portal_type)
results_dict[state_var] = state
results_dict['state_title'] = state_title
# XXX add some kind of out-of-date indicator to bika listing
## if App.config.getConfiguration().debug_mode:
## from Products.CMFEditions.utilities import dereference
## pr = getToolByName(self.context, 'portal_repository')
## o = hasattr(obj, 'getObject') and obj.getObject() or obj
## if pr.isVersionable(o):
## pa = getToolByName(self.context, 'portal_archivist')
## history_id = str(dereference(o)[1])
## version_id = hasattr(o, 'version_id') \
## and str(o.version_id) or None
## if not 'version_id' in self.columns.keys():
## self.columns['version_id'] = {'title':'version'}
## for x in range(len(self.review_states)):
## self.review_states[x]['columns'].append('version_id')
## results_dict['version_id'] = '%s/%s' % (version_id, history_id)
# extra classes for individual fields on this item { field_id : "css classes" }
results_dict['class'] = {}
# Search for values for all columns in obj
for key in self.columns.keys():
if hasattr(obj, key):
# if the key is already in the results dict
# then we don't replace it's value
if results_dict.has_key(key):
continue
value = getattr(obj, key)
if callable(value):
value = value()
results_dict[key] = value
results.append(results_dict)
return results
