def _validateDimensionsValidJson(self, identifier, informational,
dimensions):
try:
identifier_dimensions = json.loads(
identifier) if type(identifier) is str else identifier
informational_dimensions = json.loads(
informational) if type(informational) is str else informational
dimensions = json.loads(
dimensions) if type(dimensions) is str else dimensions
except Exception:
raise ArgValidationException(
'Invalid JSON supplied to REST handler!')
# Make sure the correct arg types are provided for the endpoint
if type(dimensions) is not dict:
raise ArgValidationException(
'Dimensions provided should be JSON object!')
if type(informational_dimensions) is not list:
raise ArgValidationException(
'Informational dimensions provided should be list!')
if type(identifier_dimensions) is not list:
raise ArgValidationException(
'Identifier dimensions provided should be list!')
# If all the checks pass without exception, return the dimensions as
# valid objects
return (dimensions, informational_dimensions, identifier_dimensions)
def handleBulkDelete(self, handler, confInfo):
"""
:param handler instance of the en_entity_interface:
:param confInfo confInfo for the current request:
:return:
Delete the entities based on a query and calls list as response.
Query param is required for bulk delete.
"""
self._setup_kv_store(handler)
self._setup_savedsearch_manager(handler)
kvstore_delete_query = EMCommon.get_query_from_request_args(
handler.callerArgs.data.get('delete_query', [''])[0])
if not kvstore_delete_query:
raise ArgValidationException('Delete query can not be empty')
delete_query = {'query': kvstore_delete_query}
entities_deleted_list = self._handleListAll(confInfo,
fields='_key',
query_params=delete_query)
savedsearch_delete_query = EMCommon.get_list_of_admin_managedby(
entities_deleted_list, em_constants.APP_NAME)
logger.info('User triggered action "bulk_delete" on entities')
self.entity_store.bulk_delete(query=delete_query)
self.savedsearch_manager.bulk_delete(
savedsearch_query=savedsearch_delete_query)
handler.callerArgs.data.pop('delete_query')
self.handleList(handler, confInfo)
def filter_groups_by(self,
criteria,
earliest='-24h',
latest='now',
count=0):
spl_template = self.get_search_template().get('filter_groups_by')
def by_entity_ids(entity_ids):
entity_predicate = ' OR '.join(
map(lambda e: '_key="%s"' % e, entity_ids))
return execute_spl(entity_predicate)
def by_entity_names(entity_names):
entity_predicate = ' OR '.join(
map(lambda e: 'title="%s"' % e, entity_names))
return execute_spl(entity_predicate)
def execute_spl(entity_predicate):
spl = spl_template % (EMConstants.STORE_ENTITIES, entity_predicate,
self.CUSTOM_SEARCH_CMD, EMConstants.ACTIVE,
EMConstants.INACTIVE, EMConstants.DISABLED)
results = self.search_one_shot(spl, earliest, latest, count)
return EMSearchManager.parse_group_entites_count_results(results)
if criteria == EMSearchManager.BY_ENTITY_IDS:
return by_entity_ids
elif criteria == EMSearchManager.BY_ENTITY_NAMES:
return by_entity_names
else:
raise ArgValidationException(
'Filter criteria specified is not allowed!')
def _batch_save_to_mbus(self, data, url):
"""
Perform multiple save operations in a batch
"""
if not data:
raise ArgValidationException(
_('Batch saving failed: Batch is empty.'))
batches = (data[x:x + DEFAULT_BATCH_SIZE]
for x in range(0, len(data), DEFAULT_BATCH_SIZE))
for batch in batches:
try:
payload = {
"publisher": "Splunk App for Infrastructure",
"entities": batch
}
response, content = splunk.rest.simpleRequest(
url,
method='POST',
sessionKey=session['authtoken'],
jsonargs=json.dumps(payload))
if response.status != 200:
logger.error(
"Failed to publish entities to message bus -- status:%s content:%s"
% (response.status, content))
except Exception as e:
logger.error(e)
raise e
def handleBulkDelete(self, handler, confInfo):
"""
:param handler instance of the em_group_interface:
:param confInfo confInfo for the current request:
:return:
Delete the groups based on a query and calls list as response.
Query param is required for bulk delete.
"""
self._setupKVStore(handler)
self._setupSavedSearchManager(handler)
groups_deleted_list = json.loads(
handler.callerArgs.data.get('delete_query', ['{}'])[0])
if not groups_deleted_list:
# If no query is provided, delete all groups, and therefore all saved
# searches associated with all groups
groups_deleted_list = self._handleListAll(fields='_key')
savedsearch_delete_query = EMCommon.get_list_of_admin_managedby(
groups_deleted_list, EMConstants.APP_NAME)
kvstore_delete_query = EMCommon.get_query_from_request_args(
handler.callerArgs.data.get('delete_query', [''])[0])
if not kvstore_delete_query:
raise ArgValidationException('Delete query can not be empty')
logger.info('User triggered action "bulk_delete" on groups')
delete_query = {"query": kvstore_delete_query}
self.groups_store.bulk_delete(query=delete_query)
self.savedsearch_manager.bulk_delete(
savedsearch_query=savedsearch_delete_query)
handler.callerArgs.data.pop('delete_query')
self.handleList(handler, confInfo)
def build_mongodb_query(filters, options=None):
"""
Ex1: for a input {"title": ["a*"]}
this function would return {"title": {"$regex": "a*", , "$options": "i"}}
Ex2: for a input {"title": ["a*", "b"]}
returns: {"$or": [{"title": {"$regex": "a*", "$options": "i"}}, {"title": {"$regex": "b", , "$options": "i"}} ]}
Ex3: for a input {"title": ["a*", "b"], "os": ["windows"]}
returns: {"$and": [
{"$or": [{"title": {"$regex": "a*", "$options": "i"}}, {"$regex": "b", , "$options": "i"}} ]},
{"os": {"$regex": "windows", , "$options": "i"}}}
]}
NOTE: This code needs to be updated when https://jira.splunk.com/browse/PBL-9076 is implemented.
:param filters dictionary, filters object passed by the UI
:param options dictionary of supported options
:return mongoDB format query dictionary:
"""
# if filters is empty object return as no query constructions required
if not bool(filters):
return filters
if not options:
options = _get_default_options()
elif isinstance(options, str):
options = json.loads(options)
elif not isinstance(options, dict):
raise ArgValidationException(
_('When provided, options must be string or dict'))
mongo_options = _construct_mongo_options(options)
# if filter is not a dict or options if passed in is not a dict a error will be thrown
if not isinstance(filters, dict):
raise ArgValidationException(_('Filter must be a dict'))
sub_queries = [
_construct_query_for(key, value, mongo_options)
for key, value in filters.items()
]
# if number of sub-queries is 1 return else wrap it around a "$and"
return sub_queries[0] if len(sub_queries) == 1 else {"$and": sub_queries}
def _load_valid_metric_names_query_param(self, query_param):
"""
Query params are expected to be a dictionary with dimension name as key, list of dimension values as value
"""
message = (
'Cannot parse query parameter: %s. ' % query_param +
'Expected format is {<dimension name>: [ <dimension values, wildcards>]}'
)
# Check if it's a valid json string
try:
query_param = json.loads(query_param)
except:
raise ArgValidationException(message)
if isinstance(query_param, dict):
# Check if key is string and value is list
is_query_param_valid = all(
isinstance(key, basestring) and isinstance(value, list)
for key, value in query_param.items())
if is_query_param_valid is False:
raise ArgValidationException(message)
else:
raise ArgValidationException(message)
return query_param
def convert_query_params_to_mongoDB_query(query=None, options=None):
"""
Converts query params to mongoDB format
:param query: dict or string, the query containing filter to be converted to mongo format
:param options: dict or string, options for mongo search (example: is search case sensitive or not)
:return: dict or nothing
"""
if not query:
return {}
if isinstance(query, str):
query = json.loads(query)
try:
return build_mongodb_query(query, options=options)
# Throws ArgalidationException when we provide invalid dict for mongodb query, a cleaner
# exception message than default ValueError returned
except ValueError:
raise ArgValidationException(
_('Invalid dict supplied as part of query!'))
def handleMetricData(self, handler, confInfo):
"""
Return metric metadata by entity name
"""
count = handler.callerArgs.get('count', 0)
query_params = handler.callerArgs.data.get('query', '')
if not query_params:
raise ArgValidationException('Missing required key: query')
query_params = self._load_valid_metric_metadata_query(query_params[0])
self._setup_kv_store(handler)
dimensions = query_params.get('dimensions', {})
execute_search = normalizeBoolean(
query_params.get('executeSearch', True))
reformated_dimensions = dimensions
if dimensions:
reformated_dimensions = {
'dimensions.{}'.format(key): value
for key, value in dimensions.iteritems()
}
kvstore_query = EMCommon.get_query_from_request_args(
json.dumps(reformated_dimensions))
filtered_entities = self._handleListAll(
confInfo,
fields='_key,dimensions,collectors.name',
query_params={'query': kvstore_query})
collectors = self._handleListAllConfigs(confInfo,
fields='name,title_dimension')
collector_config = {
collector.get('name'): collector.get('title_dimension')
for collector in collectors
}
search_manager = EMSearchManager(EMCommon.get_server_uri(),
handler.getSessionKey(),
em_constants.APP_NAME)
search_res = search_manager.get_avg_metric_val_by_entity(
execute_search=execute_search,
metric_name=query_params['metric_name'],
entities=filtered_entities,
collector_config=collector_config,
count=count)
confInfo['metric_data']['metric_data'] = \
json.dumps({
ret.get('key'): ret.get('value') for ret in search_res
}) if isinstance(search_res, list) else search_res
def _construct_query_for(key, value, options):
"""
If values is a list it would return
{"$or": [{"key":"a"}, {"key": "b"}]}
else it would return
{"key": "value"}
:return:
"""
if not (isinstance(value, list) or isinstance(value, basestring)):
raise ArgValidationException(_('Value needs to be a string or list'))
item = {}
if isinstance(value, list):
item['$or'] = [{
key: get_regex_search_string(v, options)
} for v in value]
else:
item[key] = get_regex_search_string(value, options)
return item
def get_list_of_admin_managedby(query, app_name):
"""
Converts UI query to a list of entities
that are preceded by 'alert.managedBy:'
:param query query string from UI
:return
"""
if not query:
return []
else:
try:
type_ids = []
if type(query) is dict:
type_ids = query.get('_key', [])
# Getting a delete all call here, so take all entities
else:
type_ids = [entity_type.get('_key') for entity_type in query]
return [
'alert.managedBy="%s:%s"' % (app_name, type_id)
for type_id in type_ids
]
except ValueError:
raise ArgValidationException(
_('Invalid JSON supplied as part of query!'))
def _load_valid_metric_metadata_query(self, query_param):
# {metric_name:cpu.idle, dimensions:{os:["ubuntu"]}}
message = (
'Cannot parse query parameter: %s. ' % query_param +
'Expected format is {metric_name: [metric_name], ' +
'dimensions: {<dimension name>: [ <dimension values, wildcards>]}}'
)
# Check if it's a valid json string
try:
query_param = json.loads(query_param)
except:
raise ArgValidationException(message)
if isinstance(query_param, dict):
# Check if both metric_name and dimensions exist
if 'metric_name' not in query_param:
raise ArgValidationException(
'Missing required key: metric_name')
metric_name = query_param['metric_name']
dimensions = query_param.get('dimensions')
# Check type for required key - metric_name
if not isinstance(metric_name, basestring):
raise ArgValidationException(
'Expected metric name to be a string.')
if dimensions:
# Check type for optional key - dimensions
if not isinstance(dimensions, dict):
raise ArgValidationException(
'Expected dimensions to be a dict.')
# Check if each key in dimensions is a string and each value is a list
is_query_param_valid = all(
isinstance(key, basestring) and isinstance(value, list)
for key, value in dimensions.iteritems())
if is_query_param_valid is False:
raise ArgValidationException(
'Expected each key in dimensions to be a string, each value to be a list'
)
else:
raise ArgValidationException('Expected query param to be a dict')
return query_param
def _validateDimensionsExist(self, dimensionsObj, dimensionList):
for dimension in dimensionList:
if dimension not in dimensionsObj:
message = 'Dimension "%s" provided does not exist in dimensions object!' % dimension
raise ArgValidationException(message)
def _validateEntityState(self, entity_state):
if entity_state not in self.VALID_ENTITY_STATES:
raise ArgValidationException('Invalid entity state: must be %s' %
self.VALID_ENTITY_STATES)