def parse_entities_and_dimensions_in_group(results, group_name): res = { group_name: { 'entities_mapping': {}, 'count': len(results['results']), 'active': 0, 'inactive': 0, 'disabled': 0 } } for record in results['results']: """ get the key and the identifier dimensions of each entity in the following format: {xyz:['blah']} """ identifier_dimensions_map = {} identifier_dimensions = em_common.always_list( record.get('identifier_dimensions', [])) for each_id_dim in identifier_dimensions: identifier_dimensions_map[each_id_dim] = em_common.always_list( record.get('dimensions.{0}'.format(each_id_dim))) res[group_name]['entities_mapping'][record.get('title')] = { 'key': record.get('key'), 'identifier_dimensions': identifier_dimensions_map, 'state': record.get('state'), } res[group_name][record.get('state')] += 1 return res
def get_dimension_names_by_id_dims(self, predicate=[], id_dims_name=[], earliest='-24h', latest='now', count=0): """ Get dimension names by identifier_dimensions :param predicate: What metric to search for. i.e. cpu.* (All metric has metric_name starts by cpu.*) :param id_dims_name: List of dimensions name to identify entity i.e. ['ip', 'host'] :param earliest: earliest time :param latest: latest time :param count: limit number of result :return: [{dims:['os','tag']}, {dims:['os','env']}] """ iden_fields = set(id_dims_name) fields_part = '' for iden in iden_fields: fields_part += 'values("%s") ' % iden iden_fields = ['"%s"' % d for d in iden_fields] fields_list = ', '.join(iden_fields) metric_name_filter = ' OR '.join( ['metric_name="' + x + '"' for x in predicate]) spl = ( '| mcatalog %s, values("_dims") as "dims" WHERE (%s) AND (`sai_metrics_indexes`) BY %s ' '| table dims') % (fields_part, metric_name_filter, fields_list) results = self.search(spl, earliest, latest, count) res = [] if 'results' in results and len(results['results']) > 0: res = results['results'] logger.info('Retrieved dimension names by identifier dimensions') return em_common.always_list(res)
def get_all_dims_from_dims_name(self, predicate='', id_dims_name=[], dims_name=[], earliest='-24h', latest='now', count=0): """ Get list of dimensions name-value for all entities :param predicate: What metric to search for. i.e. cpu.* (All metric has metric_name starts by cpu.*) :param id_dims_name: Set of dimensions to identify entity i.e. ['ip','host'] :pram dims_name: Set of dimensions name to search for i.e. ['ip', 'db_instance_id','host'] :param earliest: earliest time :param latest: latest time :param count: limit number of result :return: list of dimensions for all entities """ id_predicate = ','.join(id_dims_name) values_part = '' for d in dims_name: if d not in id_dims_name: values_part += 'values(%s) as %s ' % (d, d) spl = '| mcatalog %s WHERE metric_name=%s BY %s' % ( values_part, predicate, id_predicate) results = self.search_one_shot(spl, earliest, latest, count) res = [] if 'results' in results and len(results['results']) > 0: res = results['results'] logger.info('Received dimensions for all entities') return em_common.always_list(res)
def get_dimension_names_by_id_dims(self, predicate='', id_dims_name=[], earliest='-24h', latest='now', count=0): """ Get dimension names by identifier_dimensions :param predicate: What metric to search for. i.e. cpu.* (All metric has metric_name starts by cpu.*) :param id_dims_name: List of dimensions name to identify entity i.e. ['ip', 'host'] :param earliest: earliest time :param latest: latest time :param count: limit number of result :return: [{dims:['os','tag']}, {dims:['os','env']}] """ iden_fields = set(id_dims_name) fields_part = '' for iden in iden_fields: fields_part += 'values(%s) ' % iden fields_list = ', '.join(iden_fields) spl = '| mcatalog %s, values(_dims) as dims WHERE metric_name=%s BY %s | table dims' % ( fields_part, predicate, fields_list) results = self.search_one_shot(spl, earliest, latest, count) res = [] if 'results' in results and len(results['results']) > 0: res = results['results'] logger.info('Retrieved dimension names by identifier dimensions') return em_common.always_list(res)
def get_entities_and_collector_names_by_group(self, group_name, earliest='-24h', latest='now', count=0): """ get entities and collector information about a group :param group_name: str, the id/name of the group (not title) :param earliest: earliest time :param latest: latest time :param count: limit number of result :return: a dictionary with keys being collector names & values being the list of entities in this group and were discovered by this collector. example: { 'os': [ { 'dimensions': {'location': 'seattle', 'tag': 'planets', 'host': 'mars.planets.com', ...}}, { 'dimensions': {'location': 'seattle', 'tag': 'planets', 'host': 'mars.planets.com', ...}} ], 'aws_cloudwatch_ec2': [ { <dimensions about an ec2 instance entity>.... }, { .... } ] } """ spl_template = self.get_search_template().get( 'entities_and_collector_names_by_group') spl = spl_template % (EMConstants.STORE_ENTITIES, self.CUSTOM_SEARCH_CMD, group_name) results = self.search_one_shot(spl, earliest, latest, count) res = {} if 'results' in results and len(results['results']) > 0: for record in results['results']: collector_names = em_common.always_list( record.get('collector_names', [])) for collector_name in collector_names: entity_info = {'dimensions': {}} for dim, val in record.iteritems(): if dim.startswith('dimensions.'): entity_info['dimensions'][dim[ len('dimensions.'):]] = em_common.always_list( record[dim]) res.setdefault(collector_name, []).append(entity_info) return res
def get_metric_names_by_dim_names(self, dimensions={}, earliest='-24h', latest='now', count=0): """ Get metric names by dimension names :param dimensions: Dictionary of dimension name and values. Dimension values should support * i.e. {'location': ['seattle', 'san francisco'], 'os': ['ubuntu', '*']} :param earliest: earliest time :param latest: latest time :param count: limit number of result :return: [{metric_names:['cpu.idle','cpu.nice']}] """ # mstats requires metric_name keyword after WHERE mstats_base_command = '| mstats min(_value) as min, max(_value) as max WHERE metric_name=* AND \ (`sai_metrics_indexes`)' eval_round_command = '| eval min=round(min,2), max=round(max,2)' if not dimensions: spl = '%s BY metric_name %s' % (mstats_base_command, eval_round_command) else: filter_fields = [] for dim_name, dim_values in dimensions.items(): # First add double quotes to each dimension value dimensions[dim_name] = [ '"{}"'.format(val) for val in dim_values ] # Values of the same dimension name should be "OR" filter_fields.append(' OR '.join( '{0}={1}'.format(*dim_pair) for dim_pair in zip([dim_name] * len(dim_values), dim_values))) # Values between dimension names should be "AND" dim_filter = ' AND '.join( ['({})'.format(f) for f in filter_fields]) spl = '%s AND %s BY metric_name %s' % ( mstats_base_command, dim_filter, eval_round_command) results = self.search(spl, earliest, latest, count) res = [] if 'results' in results and len(results['results']) > 0: res = results['results'] logger.info('Retrieved metric names by dimension names') return em_common.always_list(res)
def convert_basic(self, entity): """ convert an EntityFilter to a BasicFilter with the information of the entity :param entity: dict, an entity object that carries the value of event_field :return: a BasicFilter object """ if not entity: raise ValueError('Entity info is missing for entity filter conversion.') values = em_common.always_list(entity.get('dimensions', {}).get(self.dimension_name)) if self.match_criteria == EntityFilter.ALLOWED_MATCH_CRITERIA[1]: values = map(lambda v: '*%s*' % v, values) return BasicFilter( type='include', field=self.event_field, values=values )
def get_all_dims_from_dims_name(self, predicate=[], id_dims_name=[], dims_name=[], earliest='-24h', latest='now', count=0): """ Get list of dimensions name-value for all entities :param predicate: What metric to search for. i.e. cpu.* (All metric has metric_name starts by cpu.*) :param id_dims_name: Set of dimensions to identify entity i.e. ['ip','host'] :pram dims_name: Set of dimensions name to search for i.e. ['ip', 'db_instance_id','host'] :param earliest: earliest time :param latest: latest time :param count: limit number of result :return: list of dimensions for all entities """ values_part = '' for d in dims_name: if d not in id_dims_name: values_part += 'values("%s") as "%s" ' % (d, d) id_dims_name = ['"%s"' % d for d in id_dims_name] metric_name_filter = ' OR '.join( ['metric_name="' + x + '"' for x in predicate]) id_predicate = ','.join(id_dims_name) if values_part: spl = '| mcatalog %s WHERE (%s) AND (`sai_metrics_indexes`) BY %s' % ( values_part, metric_name_filter, id_predicate) else: spl = '| mcatalog values(host) WHERE (%s) AND (`sai_metrics_indexes`) BY %s | table host' % ( metric_name_filter, id_predicate) results = self.search(spl, earliest, latest, count) res = [] if 'results' in results and len(results['results']) > 0: res = results['results'] logger.info('Received dimensions for all entities') return em_common.always_list(res)