def parse_entities_and_dimensions_in_group(results, group_name):
res = {
group_name: {
'entities_mapping': {},
'count': len(results['results']),
'active': 0,
'inactive': 0,
'disabled': 0
}
}
for record in results['results']:
"""
get the key and the identifier dimensions of each entity in the following
format:
{xyz:['blah']}
"""
identifier_dimensions_map = {}
identifier_dimensions = em_common.always_list(
record.get('identifier_dimensions', []))
for each_id_dim in identifier_dimensions:
identifier_dimensions_map[each_id_dim] = em_common.always_list(
record.get('dimensions.{0}'.format(each_id_dim)))
res[group_name]['entities_mapping'][record.get('title')] = {
'key': record.get('key'),
'identifier_dimensions': identifier_dimensions_map,
'state': record.get('state'),
}
res[group_name][record.get('state')] += 1
return res
def get_dimension_names_by_id_dims(self,
predicate=[],
id_dims_name=[],
earliest='-24h',
latest='now',
count=0):
"""
Get dimension names by identifier_dimensions
:param predicate: What metric to search for.
i.e. cpu.* (All metric has metric_name starts by cpu.*)
:param id_dims_name: List of dimensions name to identify entity
i.e. ['ip', 'host']
:param earliest: earliest time
:param latest: latest time
:param count: limit number of result
:return: [{dims:['os','tag']}, {dims:['os','env']}]
"""
iden_fields = set(id_dims_name)
fields_part = ''
for iden in iden_fields:
fields_part += 'values("%s") ' % iden
iden_fields = ['"%s"' % d for d in iden_fields]
fields_list = ', '.join(iden_fields)
metric_name_filter = ' OR '.join(
['metric_name="' + x + '"' for x in predicate])
spl = (
'| mcatalog %s, values("_dims") as "dims" WHERE (%s) AND (`sai_metrics_indexes`) BY %s '
'| table dims') % (fields_part, metric_name_filter, fields_list)
results = self.search(spl, earliest, latest, count)
res = []
if 'results' in results and len(results['results']) > 0:
res = results['results']
logger.info('Retrieved dimension names by identifier dimensions')
return em_common.always_list(res)
def get_all_dims_from_dims_name(self,
predicate='',
id_dims_name=[],
dims_name=[],
earliest='-24h',
latest='now',
count=0):
"""
Get list of dimensions name-value for all entities
:param predicate: What metric to search for.
i.e. cpu.* (All metric has metric_name starts by cpu.*)
:param id_dims_name: Set of dimensions to identify entity
i.e. ['ip','host']
:pram dims_name: Set of dimensions name to search for
i.e. ['ip', 'db_instance_id','host']
:param earliest: earliest time
:param latest: latest time
:param count: limit number of result
:return: list of dimensions for all entities
"""
id_predicate = ','.join(id_dims_name)
values_part = ''
for d in dims_name:
if d not in id_dims_name:
values_part += 'values(%s) as %s ' % (d, d)
spl = '| mcatalog %s WHERE metric_name=%s BY %s' % (
values_part, predicate, id_predicate)
results = self.search_one_shot(spl, earliest, latest, count)
res = []
if 'results' in results and len(results['results']) > 0:
res = results['results']
logger.info('Received dimensions for all entities')
return em_common.always_list(res)
def get_dimension_names_by_id_dims(self,
predicate='',
id_dims_name=[],
earliest='-24h',
latest='now',
count=0):
"""
Get dimension names by identifier_dimensions
:param predicate: What metric to search for.
i.e. cpu.* (All metric has metric_name starts by cpu.*)
:param id_dims_name: List of dimensions name to identify entity
i.e. ['ip', 'host']
:param earliest: earliest time
:param latest: latest time
:param count: limit number of result
:return: [{dims:['os','tag']}, {dims:['os','env']}]
"""
iden_fields = set(id_dims_name)
fields_part = ''
for iden in iden_fields:
fields_part += 'values(%s) ' % iden
fields_list = ', '.join(iden_fields)
spl = '| mcatalog %s, values(_dims) as dims WHERE metric_name=%s BY %s | table dims' % (
fields_part, predicate, fields_list)
results = self.search_one_shot(spl, earliest, latest, count)
res = []
if 'results' in results and len(results['results']) > 0:
res = results['results']
logger.info('Retrieved dimension names by identifier dimensions')
return em_common.always_list(res)
def get_entities_and_collector_names_by_group(self,
group_name,
earliest='-24h',
latest='now',
count=0):
"""
get entities and collector information about a group
:param group_name: str, the id/name of the group (not title)
:param earliest: earliest time
:param latest: latest time
:param count: limit number of result
:return: a dictionary with keys being collector names & values being the list of entities in
this group and were discovered by this collector.
example:
{ 'os': [
{ 'dimensions': {'location': 'seattle', 'tag': 'planets', 'host': 'mars.planets.com', ...}},
{ 'dimensions': {'location': 'seattle', 'tag': 'planets', 'host': 'mars.planets.com', ...}}
],
'aws_cloudwatch_ec2': [
{ <dimensions about an ec2 instance entity>.... },
{ .... }
]
}
"""
spl_template = self.get_search_template().get(
'entities_and_collector_names_by_group')
spl = spl_template % (EMConstants.STORE_ENTITIES,
self.CUSTOM_SEARCH_CMD, group_name)
results = self.search_one_shot(spl, earliest, latest, count)
res = {}
if 'results' in results and len(results['results']) > 0:
for record in results['results']:
collector_names = em_common.always_list(
record.get('collector_names', []))
for collector_name in collector_names:
entity_info = {'dimensions': {}}
for dim, val in record.iteritems():
if dim.startswith('dimensions.'):
entity_info['dimensions'][dim[
len('dimensions.'):]] = em_common.always_list(
record[dim])
res.setdefault(collector_name, []).append(entity_info)
return res
def get_metric_names_by_dim_names(self,
dimensions={},
earliest='-24h',
latest='now',
count=0):
"""
Get metric names by dimension names
:param dimensions: Dictionary of dimension name and values.
Dimension values should support *
i.e. {'location': ['seattle', 'san francisco'], 'os': ['ubuntu', '*']}
:param earliest: earliest time
:param latest: latest time
:param count: limit number of result
:return: [{metric_names:['cpu.idle','cpu.nice']}]
"""
# mstats requires metric_name keyword after WHERE
mstats_base_command = '| mstats min(_value) as min, max(_value) as max WHERE metric_name=* AND \
(`sai_metrics_indexes`)'
eval_round_command = '| eval min=round(min,2), max=round(max,2)'
if not dimensions:
spl = '%s BY metric_name %s' % (mstats_base_command,
eval_round_command)
else:
filter_fields = []
for dim_name, dim_values in dimensions.items():
# First add double quotes to each dimension value
dimensions[dim_name] = [
'"{}"'.format(val) for val in dim_values
]
# Values of the same dimension name should be "OR"
filter_fields.append(' OR '.join(
'{0}={1}'.format(*dim_pair)
for dim_pair in zip([dim_name] *
len(dim_values), dim_values)))
# Values between dimension names should be "AND"
dim_filter = ' AND '.join(
['({})'.format(f) for f in filter_fields])
spl = '%s AND %s BY metric_name %s' % (
mstats_base_command, dim_filter, eval_round_command)
results = self.search(spl, earliest, latest, count)
res = []
if 'results' in results and len(results['results']) > 0:
res = results['results']
logger.info('Retrieved metric names by dimension names')
return em_common.always_list(res)
def convert_basic(self, entity):
"""
convert an EntityFilter to a BasicFilter with the information of the entity
:param entity: dict, an entity object that carries the value of event_field
:return: a BasicFilter object
"""
if not entity:
raise ValueError('Entity info is missing for entity filter conversion.')
values = em_common.always_list(entity.get('dimensions', {}).get(self.dimension_name))
if self.match_criteria == EntityFilter.ALLOWED_MATCH_CRITERIA[1]:
values = map(lambda v: '*%s*' % v, values)
return BasicFilter(
type='include',
field=self.event_field,
values=values
)
def get_all_dims_from_dims_name(self,
predicate=[],
id_dims_name=[],
dims_name=[],
earliest='-24h',
latest='now',
count=0):
"""
Get list of dimensions name-value for all entities
:param predicate: What metric to search for.
i.e. cpu.* (All metric has metric_name starts by cpu.*)
:param id_dims_name: Set of dimensions to identify entity
i.e. ['ip','host']
:pram dims_name: Set of dimensions name to search for
i.e. ['ip', 'db_instance_id','host']
:param earliest: earliest time
:param latest: latest time
:param count: limit number of result
:return: list of dimensions for all entities
"""
values_part = ''
for d in dims_name:
if d not in id_dims_name:
values_part += 'values("%s") as "%s" ' % (d, d)
id_dims_name = ['"%s"' % d for d in id_dims_name]
metric_name_filter = ' OR '.join(
['metric_name="' + x + '"' for x in predicate])
id_predicate = ','.join(id_dims_name)
if values_part:
spl = '| mcatalog %s WHERE (%s) AND (`sai_metrics_indexes`) BY %s' % (
values_part, metric_name_filter, id_predicate)
else:
spl = '| mcatalog values(host) WHERE (%s) AND (`sai_metrics_indexes`) BY %s | table host' % (
metric_name_filter, id_predicate)
results = self.search(spl, earliest, latest, count)
res = []
if 'results' in results and len(results['results']) > 0:
res = results['results']
logger.info('Received dimensions for all entities')
return em_common.always_list(res)
