def _load_valid_metric_data_query(self, query_param): # {metric_name:cpu.idle, dimensions:{os:["ubuntu"]}} message = _( 'Cannot parse query parameter. Expected format is {metric_name: <metric_name>, ' 'dimensions: {<dimension name>: [<dimension values, wildcards>]}}') # Check if it's a valid json string try: query_param = json.loads(query_param) except Exception as e: logger.error( 'Failed to parse query parameters - query: %s, error: %s' % (query_param, e)) raise EntityArgValidationException(message) if isinstance(query_param, dict): # Check if both metric_name and dimensions exist if 'metric_name' not in query_param: raise EntityArgValidationException( _('Missing required key: metric_name')) metric_name = query_param['metric_name'] dimensions = query_param.get('dimensions') # Check type for required key - metric_name if not isinstance(metric_name, basestring): raise EntityArgValidationException( _('Expected metric name to be a string.')) if dimensions: self._validate_dimensions_query(dimensions) else: raise EntityArgValidationException( _('Expected query param to be a dict')) return query_param
def validate(self): for criteria in self.email_when: if criteria not in EMEmailAlertAction.SUPPORTED_EMAIL_NOTIFICATION_CRITERIA: raise AlertActionInvalidArgsException( _('Email notification criteria is not one of the supported criteria') ) if not self.email_to or len(self.email_to) == 0: raise AlertActionInvalidArgsException(_('No email_to provided')) for email_address in self.email_to: if not email_address or not re.match(r"[^@]+@[^@]+\.[^@]+", email_address): raise AlertActionInvalidArgsException(_('Invalid email address provided'))
def _validate_dimensions_query(self, dimensions): # Check type for dimensions if not isinstance(dimensions, dict): raise EntityArgValidationException( _('Expected dimensions to be a dict.')) # Check if each key in dimensions is a string and each value is a list is_query_param_valid = all( isinstance(key, basestring) and isinstance(value, list) for (key, value) in dimensions.items()) if is_query_param_valid is False: raise EntityArgValidationException( _('Expected each key in dimensions to be a string, each value to be a list' ))
def validate_format(name, api_key, routing_key): # validate name if not name: raise VictorOpsInvalidArgumentsException(_('Name cannot be empty')) # validate that the API key api_key_valid = re.search(em_constants.VICTOROPS_API_KEY, api_key) if not api_key_valid: raise VictorOpsInvalidArgumentsException(_('Invalid API key')) # validate the routing key routing_key_valid = re.search(em_constants.VICTOROPS_ROUTING_KEY, routing_key) if not routing_key_valid: raise VictorOpsInvalidArgumentsException(_('Invalid routing key'))
def _get_request_handler_and_args(self, path, method): matched_handler = self._routes.match('/' + path if path != '' else path) route_handler = matched_handler.pop( 'handler') if matched_handler else None if route_handler: if method.lower() not in route_handler.allowed_methods: raise BaseRestException( http.client.BAD_REQUEST, _('REST method %(method)s not allowed') % {'method': method}) return route_handler, matched_handler else: raise BaseRestException(http.client.NOT_FOUND, _('Not Found'))
def _load_valid_metric_names_query_param(self, query_param): """ Query params are expected to be a dictionary with dimension name as key, list of dimension values as value """ message = _( 'Cannot parse query parameter. Expected format is {<dimension name>: ' '[ <dimension values, wildcards>]}') # Check if it's a valid json string try: query_param = json.loads(query_param) except Exception as e: logger.error( 'Failed to parse query parameters - query: %s, error: %s' % (query_param, e)) raise EntityArgValidationException(message) if isinstance(query_param, dict): # Check if key is string and value is list is_query_param_valid = all( isinstance(key, basestring) and isinstance(value, list) for (key, value) in query_param.items()) if is_query_param_valid is False: raise EntityArgValidationException(message) else: raise EntityArgValidationException(message) return query_param
def _response(self, status, payload): if status not in HTTP_STATUS_CODES: logger.error('Invalid status code %s - payload: %s' % (status, payload)) return self._response_error(http.client.INTERNAL_SERVER_ERROR, _('Internal Server Error')) if status >= 400: if not isinstance(payload, basestring): logger.error( 'Invalid error message type %s as payload, must be basestring' % type(payload)) return self._response_error(http.client.INTERNAL_SERVER_ERROR, _('Internal Server Error')) return self._response_error(status, payload) else: return {'status': status, 'payload': payload}
def extract_args(self, in_string): try: args = json.loads(in_string) return Request.build_from_splunkd_request(args) except Exception as e: logger.error('Failed to build request object - error: %s' % e) raise BaseRestException(http.client.BAD_REQUEST, _('Bad Request'))
def _validate_thresholds(self, threshold_types): """ Validates the min max values """ for threshold_type in threshold_types: values = [] values.append(getattr(self, '%s_min' % threshold_type, None)) values.append(getattr(self, '%s_max' % threshold_type, None)) if not (all(v is not None for v in values) or all(v is None for v in values)): raise ThresholdInvalidArgsException( _('Invalid values for thresholds provided')) if values[0] is not None and values[0] > values[1]: raise ThresholdInvalidArgsException( _('Threshold min can not be greater than max'))
def validate(self): for criteria in self.slack_when: if criteria not in EMSlackAlertAction.SUPPORTED_SLACK_NOTIFICATION_CRITERIA: raise AlertActionInvalidArgsException( _('Slack notification criteria is not one of the supported criteria') ) EMSlack.validate_webhook_url_format(self.webhook_url)
def _check_key_validity(self, api_key, routing_key): try: EmVictorOpsInterfaceImpl._send_test_notification(api_key, routing_key) except Exception: raise VictorOpsInvalidArgumentsException( _('Credentials could not be authenticated with VictorOps') )
def _batch_save_to_mbus(self, data, url): """ Perform multiple save operations in a batch """ if not data: raise ArgValidationException( _('Batch saving failed: Batch is empty.')) batches = (data[x:x + DEFAULT_BATCH_SIZE] for x in range(0, len(data), DEFAULT_BATCH_SIZE)) for batch in batches: try: payload = { "publisher": "Splunk App for Infrastructure", "entities": batch } response, content = splunk.rest.simpleRequest( url, method='POST', sessionKey=session['authtoken'], jsonargs=json.dumps(payload)) if response.status != 200: logger.error( "Failed to publish entities to message bus -- status:%s content:%s" % (response.status, content)) except Exception as e: logger.error(e) raise e
def handle_create(self, request): # Pull the data we will use to create a VO setting name, api_key, routing_key = self._retrieve_vo_args(request) # check for any VO setting existence all_vos = EMVictorOps.load() if len(all_vos): raise VictorOpsUsernameAlreadyExistsException( _('Cannot create more than one VictorOps settings') ) # check for duplicate VO setting existence existing_vo = EMVictorOps.get(name) if existing_vo: raise VictorOpsUsernameAlreadyExistsException( _('VictorOps setting with name %(name)s already exists') ) new_vo = EMVictorOps.create(name, api_key, routing_key) return new_vo
def handle_get_alert(self, request, alert_name): search_result = self.savedsearch_manager.get(alert_name) if search_result is not None: entry = self._parse_search_response(search_result) if len(list(entry)) == 1: return entry[0] raise AlertNotFoundException( _('Unable to get alert definition with given name'))
def handle_remove(self, request, vo_id): vo = EMVictorOps.get(vo_id) if not vo: raise VictorOpsNotExistException( _('VictorOps setting with name %(vo_id)s does not exist') ) vo.delete() return True
def build_mongodb_query(filters, options=None): """ Ex1: for a input {"title": ["a*"]} this function would return {"title": {"$regex": "a*", , "$options": "i"}} Ex2: for a input {"title": ["a*", "b"]} returns: {"$or": [{"title": {"$regex": "a*", "$options": "i"}}, {"title": {"$regex": "b", , "$options": "i"}} ]} Ex3: for a input {"title": ["a*", "b"], "os": ["windows"]} returns: {"$and": [ {"$or": [{"title": {"$regex": "a*", "$options": "i"}}, {"$regex": "b", , "$options": "i"}} ]}, {"os": {"$regex": "windows", , "$options": "i"}}} ]} NOTE: This code needs to be updated when https://jira.splunk.com/browse/PBL-9076 is implemented. :param filters dictionary, filters object passed by the UI :param options dictionary of supported options :return mongoDB format query dictionary: """ # if filters is empty object return as no query constructions required if not bool(filters): return filters if not options: options = _get_default_options() elif isinstance(options, str): options = json.loads(options) elif not isinstance(options, dict): raise ArgValidationException( _('When provided, options must be string or dict')) mongo_options = _construct_mongo_options(options) # if filter is not a dict or options if passed in is not a dict a error will be thrown if not isinstance(filters, dict): raise ArgValidationException(_('Filter must be a dict')) sub_queries = [ _construct_query_for(key, value, mongo_options) for key, value in filters.items() ] # if number of sub-queries is 1 return else wrap it around a "$and" return sub_queries[0] if len(sub_queries) == 1 else {"$and": sub_queries}
def handle_edit(self, request, vo_id): _unused, api_key, routing_key = self._retrieve_vo_args(request) vo = EMVictorOps.get(vo_id) if not vo: raise VictorOpsNotExistException( _('VictorOps setting with name %(vo_id)s does not exist') ) vo.update(api_key, routing_key) return vo
def _get_entity_filter_query(self, request): query = request.query.get('query', '{}') query_dict = None try: query_dict = json.loads(query) except ValueError: raise EntityArgValidationException( _('Invalid query format, expected JSON')) return query_dict
def handle_delete_alert(self, request, alert_name): self.handle_get_alert(request, alert_name) try: self.savedsearch_manager.delete(alert_name) except Exception as e: logger.error('Failed to delete alert %s - error: %s' % (alert_name, e)) raise AlertInternalException( _('Cannot delete alert with name %(alert_name)s'))
def handle_get(self, request, key): entity = EmEntity.get(key) if not entity: raise EntityNotFoundException( _('Entity with id %(key)s not found.')) correlation_filter = entity.get_correlation_filter() response = self.extract_entity_json_response(entity) response.update({'correlation_filter': serialize(correlation_filter)}) return response
def handle_update_alert(self, request, alert_name): data = {'name': alert_name} data.update(request.data) alert = self._build_alert(data) alert_params = alert.to_params() # name in data body should be only used for creation del alert_params['name'] response = self.savedsearch_manager.update(alert_name, alert_params) entry = self._parse_search_response(response) if len(entry) == 1: return entry[0] raise AlertInternalException(_('Unable to update alert'))
def handle_metric_data(self, request): count = request.query.get('count', 0) query = request.query.get('query', '') if not query: raise EntityArgValidationException( _('Missing required query parameter: query')) query_params = self._load_valid_metric_data_query(query) dimensions = query_params.get('dimensions', {}) dimensions = { 'dimensions.{}'.format(key): value for key, value in dimensions.items() } # retrieve filtered entities and transform it for get_avg_metric_val_by_entity() filtered_entities = EmEntity.load(count, 0, '', 'asc', dimensions) filtered_entities = [{ "key": entity.key, "collectors": [{ "name": entity.entity_class }], "dimensions": { dim_key: dim_val for (dim_key, dim_vals) in entity.dimensions.items() for dim_val in dim_vals } } for entity in filtered_entities] # get entity class map of key to title dimension entity_classes = EntityClass.load() collectors_map = { ec.key: { 'title_dim': ec.title_dimension, 'id_dims': ec.identifier_dimensions } for ec in entity_classes } # run search should_execute_search = normalizeBoolean( query_params.get('executeSearch', True)) search_res = self.search_manager.get_avg_metric_val_by_entity( execute_search=should_execute_search, metric_name=query_params['metric_name'], entities=filtered_entities, collector_config=collectors_map, count=count, collection=em_constants.STORE_ENTITY_CACHE) response = {res.get('key'): res.get('value') for res in search_res} if isinstance(search_res, list) else search_res return response
def handle_create_alert(self, request): alert = self._build_alert(request.data) existing_alert = None try: existing_alert = self.savedsearch_manager.get(alert.name) except Exception: pass if existing_alert is not None: raise AlertAlreadyExistsException( _('Alert with the specified name already exists.')) response = self.savedsearch_manager.create(alert.to_params()) entry = self._parse_search_response(response) if len(entry) == 1: return entry[0]
def convert_query_params_to_mongoDB_query(query=None, options=None): """ Converts query params to mongoDB format :param query: dict or string, the query containing filter to be converted to mongo format :param options: dict or string, options for mongo search (example: is search case sensitive or not) :return: dict or nothing """ if not query: return {} if isinstance(query, str): query = json.loads(query) try: return build_mongodb_query(query, options=options) # Throws ArgalidationException when we provide invalid dict for mongodb query, a cleaner # exception message than default ValueError returned except ValueError: raise ArgValidationException( _('Invalid dict supplied as part of query!'))
def _construct_query_for(key, value, options): """ If values is a list it would return {"$or": [{"key":"a"}, {"key": "b"}]} else it would return {"key": "value"} :return: """ if not (isinstance(value, list) or isinstance(value, basestring)): raise ArgValidationException(_('Value needs to be a string or list')) item = {} if isinstance(value, list): item['$or'] = [{ key: get_regex_search_string(v, options) } for v in value] else: item[key] = get_regex_search_string(value, options) return item
def handle(self, in_string): ''' Implementation of handle method defined in PersistentServerConnectionApplication. ''' request_obj = self.extract_args(in_string) try: session.save(**request_obj.session) handler, args = self._get_request_handler_and_args( request_obj.path, request_obj.method) self.execute_before_handle_hooks(request_obj) status_code, response = handler(self, request_obj, **args) return self._response(status_code, response) except BaseRestException as e: return self._response(e.code, e.msg) except Exception as e: logger.error('Invalid response - Error: %s' % e.args[0]) logger.debug(traceback.format_exc()) return self._response(http.client.INTERNAL_SERVER_ERROR, _('Internal Server Error')) finally: session.clear()
def _build_alert(self, data): """ build alert savedsearch spl :param data.name: name of the alert :param data.managed_by_id: id of the entity/group that this alert belongs to :param data.managed_by_type: type of object that manages this alert :param data.metric_spl: base SPL that extract metric data :param data.metric_filters: filters for the metrics of the alert :return: an instance of EMAlert """ missing = [] def get_with_check(param): val = data.get(param, '') if not val: missing.append(param) return val name = get_with_check('name') metric_spl = get_with_check('metric_spl') managed_by_id = get_with_check('managed_by_id') managed_by_type = get_with_check('managed_by_type') metric_filters = data.get('metric_filters', []) if len(missing) > 0: raise AlertArgValidationException( _('Invalid alert. Missing required arguments: %s' % missing)) threshold = self._build_threshold(data) actions = self._build_alert_actions(data) if not isinstance(actions, list): actions = [actions] return EMAlert( name=name, managed_by=managed_by_id, managed_by_type=managed_by_type, metric_spl=metric_spl, threshold=threshold, actions=actions, metric_filters=metric_filters, )
def get_list_of_admin_managedby(query, app_name): """ Converts UI query to a list of entities that are preceded by 'alert.managedBy:' :param query query string from UI :return """ if not query: return [] else: try: type_ids = [] if type(query) is dict: type_ids = query.get('_key', []) # Getting a delete all call here, so take all entities else: type_ids = [entity_type.get('_key') for entity_type in query] return [ 'alert.managedBy="%s:%s"' % (app_name, type_id) for type_id in type_ids ] except ValueError: raise ArgValidationException( _('Invalid JSON supplied as part of query!'))
} ENTITY_CLASS_TO_ENTITY_TYPE_IDS = { 'os': 'nix', 'telegraf': 'nix', 'perfmon': 'windows', 'k8s_node': 'k8s_node', 'k8s_pod': 'k8s_pod', 'vmware_host': 'vmware_esxi_host', 'vmware_vm': 'vmware_vm', 'vmware_cluster': 'vmware_cluster', 'vmware_vcenter': 'vmware_vcenter', } # Default metric used for color by in tile view DEFAULT_METRIC_FOR_COLOR_BY = _('Availability') # Endpoint to fetch latest created alerts LATEST_ALERTS_ENDPOINT = '%s/servicesNS/-/%s/admin/alerts/-?%s' # Endpoint to fetch metadata about created alert ALERTS_METADATA_ENDPOINT = '%s/servicesNS/-/%s/saved/searches/%s?%s' # Endpoint to fetch results via search_id SEARCH_RESULTS_ENDPOINT = '%s/servicesNS/-/%s/search/jobs/%s/results' # Regular expression to extract alerting entity and alerting metric ALERTS_SEARCH_EXTRACTION = r'\(?\"(host|InstanceId)\"=\"(?P<alerting_entity>[^\"]+)\"\)? ' \ 'AND metric_name=\"(?P<metric_name>[^\"]+)\"' # Regular expression to match routing key, from best practices at
import http.client from em_utils import get_conf_stanza from utils.i18n_py23 import _ # set up logger logger = log.getLogger() VICTOROPS_CONF_FILE = 'victorops' # default monitoring tool to use for alerts from SII MONITORING_TOOL = "splunk_sii" SPLUNK_ALERT_CODE_TO_VICTOROPS_INCIDENT_LEVEL = { "1": "INFO", "3": "WARNING", "5": "CRITICAL" } VICTOROPS_REST_API_GENERAL_ERROR_MESSAGE = _('Unable to fetch VictorOps data.') class VictorOpsUsernameAlreadyExistsException(BaseRestException): def __init__(self, message): super(VictorOpsUsernameAlreadyExistsException, self).__init__(http.client.BAD_REQUEST, message) class VictorOpsNotExistException(BaseRestException): def __init__(self, message): super(VictorOpsNotExistException, self).__init__(http.client.BAD_REQUEST, message) class VictorOpsInvalidArgumentsException(BaseRestException):