def handleMetricName(self, handler, confInfo):
"""
Return metric names of the entities
"""
self._setup_kv_store(handler)
search_query = handler.callerArgs.data.get('query', '')
if search_query:
search_query = self._load_valid_metric_names_query_param(
search_query[0])
count = handler.callerArgs.get('count', 0)
search_manager = EMSearchManager(EMCommon.get_server_uri(),
handler.getSessionKey(),
em_constants.APP_NAME)
search_results_list = search_manager.get_metric_names_by_dim_names(
dimensions=search_query, count=count)
metrics_list = []
if search_results_list:
for result in search_results_list:
single_metric = {
result.get('metric_name'): {
'min': result.get('min'),
'max': result.get('max')
}
}
metrics_list.append(single_metric)
# Availability should always be the first metric
metrics_list.insert(
0, {
em_constants.DEFAULT_METRIC_FOR_COLOR_BY: {
'min': '0.00',
'max': '1.00'
}
})
confInfo['metric_names']['metric_names'] = json.dumps(metrics_list)
def _setupSearchManager(self, handler):
# Sets up the search manager
logger.info('Setting up the EM Search Manager...')
self.search_manager = EMSearchManager(
server_uri=EMCommon.get_server_uri(),
session_key=handler.getSessionKey(),
app=EMConstants.APP_NAME)
def __init__(self, logger, session_key):
super(StaticEntityToDynamicEntity, self).__init__(logger, session_key)
server_uri = em_common.get_server_uri()
self.search_manager = EMSearchManager(
server_uri=server_uri,
session_key=self.session_key,
app=em_constants.APP_NAME,
)
self.saved_search_manager = SavedSearchManager(
session_key=self.session_key,
server_uri=server_uri,
app=em_constants.APP_NAME,
)
self.static_entity_store = KVStoreManager(
collection='em_entities',
server_uri=server_uri,
session_key=self.session_key,
app=em_constants.APP_NAME,
)
self.entity_classes = {ec.key: ec for ec in EntityClass.load()}
self.new_entities = []
self.updated_alerts = {}
self.entity_key_mapping = {}
self.existing_dynamic_entities = set(
[e.key for e in EmEntity.load(0, 0, '', 'asc')])
def discover_entities(self):
"""
Discover entities from identifier dimensions
:return: list of entities
"""
search_manager = EMSearchManager(em_common.get_server_uri(),
self.session_key,
em_constants.APP_NAME)
earliest = '-%ss' % (self.monitoring_calculation_window +
self.monitoring_lag)
latest = '-%ss' % self.monitoring_lag
dims_list = search_manager.get_dimension_names_by_id_dims(
predicate=self.source_predicate,
id_dims_name=self.identifier_dimensions,
earliest=earliest,
latest=latest,
count=0)
dimension_names = []
for dims in dims_list:
dimension_names += dims.get('dims', [])
# Remove duplicates
dimension_names = list(set(dimension_names))
# Filter out black_listed dimensions
dimension_names = filter(
lambda d: d not in self.blacklisted_dimensions, dimension_names)
# | mcatalog values(_dims) doesn't return native splunk dimensions
# There are 3 native dimensions: host, source, sourcetype
# If user wants to identify entity by those host then this search
# won't work.
# Hence, we need to add host to the list as dimension.
if len(filter(lambda d: d == 'host', dimension_names)) == 0:
dimension_names += ['host']
# Get dimension name-value pairs for all entities
entities_dimensions_list = search_manager.get_all_dims_from_dims_name(
predicate=self.source_predicate,
id_dims_name=self.identifier_dimensions,
dims_name=dimension_names,
earliest=earliest,
latest=latest)
entities = []
for entity_dimensions in entities_dimensions_list:
entities.append(self.get_entity(entity_dimensions))
return entities
def load_filter_by_entity_ids(cls, entity_ids):
'''
load groups that contain entities corresponding to the input entity ids
:type entity_ids: list of str
:param entity_ids: list of entity ids
:rtype list
:return list of EMGroup objects
'''
search_manager = EMSearchManager(em_common.get_server_uri(),
session['authtoken'],
em_constants.APP_NAME)
groups_with_count = search_manager.filter_groups_by_entity_ids(
entity_ids)
group_keys = list(groups_with_count)
if len(group_keys):
return EMGroup.load(0, 0, '', 'asc', query={'_key': group_keys})
return []
def handleMetricData(self, handler, confInfo):
"""
Return metric metadata by entity name
"""
count = handler.callerArgs.get('count', 0)
query_params = handler.callerArgs.data.get('query', '')
if not query_params:
raise ArgValidationException('Missing required key: query')
query_params = self._load_valid_metric_metadata_query(query_params[0])
self._setup_kv_store(handler)
dimensions = query_params.get('dimensions', {})
execute_search = normalizeBoolean(
query_params.get('executeSearch', True))
reformated_dimensions = dimensions
if dimensions:
reformated_dimensions = {
'dimensions.{}'.format(key): value
for key, value in dimensions.iteritems()
}
kvstore_query = EMCommon.get_query_from_request_args(
json.dumps(reformated_dimensions))
filtered_entities = self._handleListAll(
confInfo,
fields='_key,dimensions,collectors.name',
query_params={'query': kvstore_query})
collectors = self._handleListAllConfigs(confInfo,
fields='name,title_dimension')
collector_config = {
collector.get('name'): collector.get('title_dimension')
for collector in collectors
}
search_manager = EMSearchManager(EMCommon.get_server_uri(),
handler.getSessionKey(),
em_constants.APP_NAME)
search_res = search_manager.get_avg_metric_val_by_entity(
execute_search=execute_search,
metric_name=query_params['metric_name'],
entities=filtered_entities,
collector_config=collector_config,
count=count)
confInfo['metric_data']['metric_data'] = \
json.dumps({
ret.get('key'): ret.get('value') for ret in search_res
}) if isinstance(search_res, list) else search_res
def __init__(self, session_key):
self.session_key = session_key
self.search_manager = EMSearchManager(em_common.get_server_uri(),
self.session_key,
em_constants.APP_NAME)
class EmEntityInterfaceImpl(object):
def __init__(self, session_key):
self.session_key = session_key
self.search_manager = EMSearchManager(em_common.get_server_uri(),
self.session_key,
em_constants.APP_NAME)
def _get_entity_filter_query(self, request):
query = request.query.get('query', '{}')
query_dict = None
try:
query_dict = json.loads(query)
except ValueError:
raise EntityArgValidationException(
_('Invalid query format, expected JSON'))
return query_dict
def handle_load(self, request):
count = request.query.get('count', 0)
offset = request.query.get('offset', 0)
sort_key = request.query.get('sort_key', '')
sort_dir = request.query.get('sort_dir', 'asc')
query_dict = self._get_entity_filter_query(request)
entities = EmEntity.load(count, offset, sort_key, sort_dir, query_dict)
return [
self.extract_entity_json_response(entity) for entity in entities
]
def handle_get(self, request, key):
entity = EmEntity.get(key)
if not entity:
raise EntityNotFoundException(
_('Entity with id %(key)s not found.'))
correlation_filter = entity.get_correlation_filter()
response = self.extract_entity_json_response(entity)
response.update({'correlation_filter': serialize(correlation_filter)})
return response
def handle_delete(self, request, key):
query = {'_key': [key]}
EmEntity.bulk_delete(query)
def handle_bulk_delete(self, request):
query = self._get_entity_filter_query(request)
exclusion_list = json.loads(request.query.get('exclusion_list', '[]'))
EmEntity.bulk_delete(query, exclusion_list=exclusion_list)
def handle_metadata(self, request):
query_dict = self._get_entity_filter_query(request)
metadata = EmEntity.get_metadata(query_dict)
return metadata
def handle_dimension_summary(self, request):
query_dict = self._get_entity_filter_query(request)
dim_summary = EmEntity.get_dimension_summary(query_dict)
return {'dimensions': dim_summary}
def handle_metric_names(self, request):
count = request.query.get('count', 0)
query = request.query.get('query')
if query:
query = self._load_valid_metric_names_query_param(query)
results_list = self.search_manager.get_metric_names_by_dim_names(
dimensions=query, count=count)
metrics_list = [{
em_constants.DEFAULT_METRIC_FOR_COLOR_BY: {
'min': '0.00',
'max': '1.00'
}
}]
if results_list:
for r in results_list:
metrics_list.append({
r.get('metric_name'): {
'min': r.get('min'),
'max': r.get('max')
}
})
return metrics_list
def handle_metric_data(self, request):
count = request.query.get('count', 0)
query = request.query.get('query', '')
if not query:
raise EntityArgValidationException(
_('Missing required query parameter: query'))
query_params = self._load_valid_metric_data_query(query)
dimensions = query_params.get('dimensions', {})
dimensions = {
'dimensions.{}'.format(key): value
for key, value in dimensions.items()
}
# retrieve filtered entities and transform it for get_avg_metric_val_by_entity()
filtered_entities = EmEntity.load(count, 0, '', 'asc', dimensions)
filtered_entities = [{
"key": entity.key,
"collectors": [{
"name": entity.entity_class
}],
"dimensions": {
dim_key: dim_val
for (dim_key, dim_vals) in entity.dimensions.items()
for dim_val in dim_vals
}
} for entity in filtered_entities]
# get entity class map of key to title dimension
entity_classes = EntityClass.load()
collectors_map = {
ec.key: {
'title_dim': ec.title_dimension,
'id_dims': ec.identifier_dimensions
}
for ec in entity_classes
}
# run search
should_execute_search = normalizeBoolean(
query_params.get('executeSearch', True))
search_res = self.search_manager.get_avg_metric_val_by_entity(
execute_search=should_execute_search,
metric_name=query_params['metric_name'],
entities=filtered_entities,
collector_config=collectors_map,
count=count,
collection=em_constants.STORE_ENTITY_CACHE)
response = {res.get('key'): res.get('value')
for res in search_res} if isinstance(search_res,
list) else search_res
return response
@staticmethod
def extract_entity_json_response(entity):
id_dims, info_dims = {}, {}
for dim, val in entity.dimensions.items():
if dim in entity.identifier_dimension_names:
id_dims[dim] = val
else:
info_dims[dim] = val
entity_class = entity.get_entity_class_info()
dimension_display_names = em_common.get_locale_specific_display_names(
entity_class.dimension_display_names)
return {
'_key': entity.key,
'title': entity.title,
'entity_class': entity.entity_class,
'mod_time': entity.mod_time,
'expiry_time': entity.expiry_time,
'status': entity.status,
'identifier_dimensions': id_dims,
'informational_dimensions': info_dims,
'vital_metrics': entity_class.vital_metrics,
'dimension_display_names': dimension_display_names
}
def _load_valid_metric_names_query_param(self, query_param):
"""
Query params are expected to be a dictionary with dimension name as key, list of dimension values as value
"""
message = _(
'Cannot parse query parameter. Expected format is {<dimension name>: '
'[ <dimension values, wildcards>]}')
# Check if it's a valid json string
try:
query_param = json.loads(query_param)
except Exception as e:
logger.error(
'Failed to parse query parameters - query: %s, error: %s' %
(query_param, e))
raise EntityArgValidationException(message)
if isinstance(query_param, dict):
# Check if key is string and value is list
is_query_param_valid = all(
isinstance(key, basestring) and isinstance(value, list)
for (key, value) in query_param.items())
if is_query_param_valid is False:
raise EntityArgValidationException(message)
else:
raise EntityArgValidationException(message)
return query_param
def _load_valid_metric_data_query(self, query_param):
# {metric_name:cpu.idle, dimensions:{os:["ubuntu"]}}
message = _(
'Cannot parse query parameter. Expected format is {metric_name: <metric_name>, '
'dimensions: {<dimension name>: [<dimension values, wildcards>]}}')
# Check if it's a valid json string
try:
query_param = json.loads(query_param)
except Exception as e:
logger.error(
'Failed to parse query parameters - query: %s, error: %s' %
(query_param, e))
raise EntityArgValidationException(message)
if isinstance(query_param, dict):
# Check if both metric_name and dimensions exist
if 'metric_name' not in query_param:
raise EntityArgValidationException(
_('Missing required key: metric_name'))
metric_name = query_param['metric_name']
dimensions = query_param.get('dimensions')
# Check type for required key - metric_name
if not isinstance(metric_name, basestring):
raise EntityArgValidationException(
_('Expected metric name to be a string.'))
if dimensions:
self._validate_dimensions_query(dimensions)
else:
raise EntityArgValidationException(
_('Expected query param to be a dict'))
return query_param
def _validate_dimensions_query(self, dimensions):
# Check type for dimensions
if not isinstance(dimensions, dict):
raise EntityArgValidationException(
_('Expected dimensions to be a dict.'))
# Check if each key in dimensions is a string and each value is a list
is_query_param_valid = all(
isinstance(key, basestring) and isinstance(value, list)
for (key, value) in dimensions.items())
if is_query_param_valid is False:
raise EntityArgValidationException(
_('Expected each key in dimensions to be a string, each value to be a list'
))
class EmGroupsInterfaceImpl(object):
""" The Groups interface that allows CRUD operations on groups """
def handleCreate(self, handler, confInfo):
# Get the name of the group the user wants to create
group_name = handler.callerArgs.id
logger.info('User triggered action "create" with key=%s' % group_name)
self._setupKVStore(handler)
data = self._setupDataPayload(handler)
try:
self.groups_store.create(group_name, data)
except Exception:
raise GroupInternalException('Failed to create the group %s!' %
group_name)
def handleList(self, handler, confInfo):
# Get the name of the group the user wants to fetch
key = handler.callerArgs.id
count = handler.callerArgs.get('count', 0)
offset = handler.callerArgs.get('offset', 0)
fields = handler.callerArgs.get('fields', '')
kvstore_query = EMCommon.get_query_from_request_args(
handler.callerArgs.get('query', [''])[0])
filter_entity_ids = handler.callerArgs.get('filter_by_entity_ids',
[''])[0]
filter_entity_names = handler.callerArgs.get('filter_by_entity_names',
[''])[0]
self._setupKVStore(handler)
self._setupSearchManager(handler)
# This will list only the group that was requested
if key is not None:
logger.info('User triggered action "list" with key=%s' % key)
selected_group = self._handleListKey(key)
if selected_group is not None:
group_entities_count = self._getNumOfEntitiesInGroups(
[selected_group])
collector_names_with_entities = self._getCollectorNamesAndEntitiesInGroup(
key)
self._extractRelevantFields(
selected_group,
group_entities_count,
confInfo,
collector_entities_match=collector_names_with_entities)
else:
# This will list groups filtered by entity ids or names
if filter_entity_ids != '' or filter_entity_names != '':
if filter_entity_ids != '':
filter_info = filter_entity_ids
group_keys, group_entities_count = self._getGroupFilteredByEntity(
EMSearchManager.BY_ENTITY_IDS, filter_info)
else:
filter_info = filter_entity_names
group_keys, group_entities_count = self._getGroupFilteredByEntity(
EMSearchManager.BY_ENTITY_NAMES, filter_info)
kvstore_query = {
'$or': [{
'_key': group_key
} for group_key in group_keys]
}
query_params = self._buildParamsObjectForLoad(
json.dumps(kvstore_query))
groups = self._handleListAll(fields, query_params, count,
offset)
else:
# This will list all of the groups saved
query_params = self._buildParamsObjectForLoad(kvstore_query)
groups = self._handleListAll(fields, query_params, count,
offset)
group_entities_count = self._getNumOfEntitiesInGroups(groups)
if groups is not None:
map(
lambda group: self._extractRelevantFields(
group, group_entities_count, confInfo), groups)
def handleEdit(self, handler, confInfo):
# Get the name of the group the user wants to fetch
group_name = handler.callerArgs.id
logger.info('User triggered action "edit" with key=%s' % group_name)
self._setupKVStore(handler)
existing_group = self.groups_store.get(group_name)
# Make sure that the group we're trying to edit exists
if existing_group is None:
raise GroupNotFoundException(
'Cannot modify a group that does not exist!')
else:
data = self._setupDataPayload(handler)
try:
self.groups_store.update(group_name, data)
except Exception:
raise GroupInternalException('Failed to update the group %s!' %
group_name)
def handleRemove(self, handler, confInfo):
# Get the name of the group the user wants to delete
group_name = handler.callerArgs.id
logger.info('User triggered action "remove" with key=%s' % group_name)
self._setupKVStore(handler)
try:
self.groups_store.delete(group_name)
except Exception:
raise GroupNotFoundException('Cannot find the group with id %s!' %
group_name)
def handleBulkDelete(self, handler, confInfo):
"""
:param handler instance of the em_group_interface:
:param confInfo confInfo for the current request:
:return:
Delete the groups based on a query and calls list as response.
Query param is required for bulk delete.
"""
self._setupKVStore(handler)
self._setupSavedSearchManager(handler)
groups_deleted_list = json.loads(
handler.callerArgs.data.get('delete_query', ['{}'])[0])
if not groups_deleted_list:
# If no query is provided, delete all groups, and therefore all saved
# searches associated with all groups
groups_deleted_list = self._handleListAll(fields='_key')
savedsearch_delete_query = EMCommon.get_list_of_admin_managedby(
groups_deleted_list, EMConstants.APP_NAME)
kvstore_delete_query = EMCommon.get_query_from_request_args(
handler.callerArgs.data.get('delete_query', [''])[0])
if not kvstore_delete_query:
raise ArgValidationException('Delete query can not be empty')
logger.info('User triggered action "bulk_delete" on groups')
delete_query = {"query": kvstore_delete_query}
self.groups_store.bulk_delete(query=delete_query)
self.savedsearch_manager.bulk_delete(
savedsearch_query=savedsearch_delete_query)
handler.callerArgs.data.pop('delete_query')
self.handleList(handler, confInfo)
def handleMetadata(self, handler, confInfo):
"""
Return metadata about the groups
"""
self._setupKVStore(handler)
fields = ["title"]
query = {"query": ""}
groups = self._handleListAll(','.join(fields), query)
confInfo["groups"]["titles"] = list(
{group["title"]
for group in groups})
def _setupSavedSearchManager(self, handler):
self.savedsearch_manager = EMSavedSearchManager(
server_uri=EMCommon.get_server_uri(),
session_key=handler.getSessionKey())
def _setupKVStore(self, handler):
# Sets up the KV store from which we will be conducting operations
logger.info('Setting up the Groups KV Store...')
self.groups_store = EMKVStoreManager(
collection=EMConstants.STORE_GROUPS,
server_uri=EMCommon.get_server_uri(),
session_key=handler.getSessionKey(),
app=EMConstants.APP_NAME)
self.collector_store = EMKVStoreManager(
collection=EMConstants.STORE_COLLECTORS,
server_uri=EMCommon.get_server_uri(),
session_key=handler.getSessionKey(),
app=EMConstants.APP_NAME)
def _setupSearchManager(self, handler):
# Sets up the search manager
logger.info('Setting up the EM Search Manager...')
self.search_manager = EMSearchManager(
server_uri=EMCommon.get_server_uri(),
session_key=handler.getSessionKey(),
app=EMConstants.APP_NAME)
def _setupDataPayload(self, handler):
data = {}
data['name'] = handler.callerArgs.id
data['title'] = handler.callerArgs.data['title'][0]
data['filter'] = handler.callerArgs.data['filter'][0]
return data
def _buildParamsObjectForLoad(self, kvstore_query):
query_params = {}
if kvstore_query:
query_params['query'] = kvstore_query
return query_params
def _extractRelevantFields(self,
group,
group_entities_count,
confInfo,
collector_entities_match=None):
confInfo[group['_key']]['filter'] = group.get('filter', '')
confInfo[group['_key']]['name'] = group.get('name', '')
confInfo[group['_key']]['title'] = group.get('title', '')
entity_state_counts = group_entities_count.get(group['_key'], {})
confInfo[group['_key']]['entities_count'] = entity_state_counts.get(
'count', 0)
confInfo[group['_key']][
'inactive_entities_count'] = entity_state_counts.get(
'inactive', 0)
confInfo[
group['_key']]['active_entities_count'] = entity_state_counts.get(
'active', 0)
confInfo[group['_key']][
'disabled_entities_count'] = entity_state_counts.get(
'disabled', 0)
confInfo[group['_key']][
'workspace_url_path'] = "/app/%s/metrics_analysis" % EMConstants.APP_NAME
if collector_entities_match:
collector_configs = self._handleListAllCollectorConfigs(
collector_entities_match.keys())
group_logs_filter = create_group_log_filter(
collector_entities_match, collector_configs)
confInfo[group['_key']]['log_search'] = serialize(
group_logs_filter) if group_logs_filter else None
def _handleListKey(self, key):
try:
return self.groups_store.get(key)
except Exception:
raise GroupNotFoundException('Cannot find the group with id %s!' %
key)
def _handleListAll(self, fields='', query_params={}, count=0, offset=0):
try:
return self.groups_store.load(count,
offset,
fields,
params=query_params)
except Exception:
raise GroupInternalException(
'Cannot list all of the groups saved!')
def _handleListAllCollectorConfigs(self,
collector_names,
fields='',
count=0,
offset=0):
try:
kvstore_query = {
'$or': [{
'_key': collector_name
} for collector_name in collector_names]
}
query_params = self._buildParamsObjectForLoad(
json.dumps(kvstore_query))
return self.collector_store.load(count,
offset,
fields,
params=query_params)
except Exception as e:
raise GroupInternalException('Cannot list collector configs: %s' %
e.message)
def _getNumOfEntitiesInGroups(self, groups):
try:
return self.search_manager.get_count_of_entities_by_group(groups)
except Exception:
raise GroupInternalException('Cannot get count of groups')
def _getCollectorNamesAndEntitiesInGroup(self, group_name):
try:
return self.search_manager.get_entities_and_collector_names_by_group(
group_name)
except Exception:
raise GroupInternalException(
'Cannot get collector or entities info from group \'%s\'' %
group_name)
def _getGroupFilteredByEntity(self,
by=EMSearchManager.BY_ENTITY_IDS,
entity_info=None):
if by == EMSearchManager.BY_ENTITY_IDS:
filter_search = self.search_manager.filter_groups_by(
EMSearchManager.BY_ENTITY_IDS)
else:
filter_search = self.search_manager.filter_groups_by(
EMSearchManager.BY_ENTITY_NAMES)
try:
entity_info_list = map(lambda e: e.strip(), entity_info.split(','))
groups_with_count = filter_search(entity_info_list)
group_keys = groups_with_count.keys()
if not len(group_keys):
raise GroupNotFoundException('No groups found.')
return group_keys, groups_with_count
except Exception:
raise GroupInternalException(
'Cannot get matching groups with given entity information')