def test_ensure_iterable(self): """Test the ensure_iterable function. """ self.assertEqual(ensure_iterable(2), [2]) self.assertEqual(ensure_iterable("2"), ["2"]) self.assertEqual(ensure_iterable([2]), [2]) self.assertEqual(ensure_iterable([2], iterable=set), {2})
def register_middleware(cls, protocol=None, data_type=None): """Register a middleware class with input protocol name and data_type. :param protocol: one or many protocols to add to protocols of input cls :type protocol: str or Iterable(protocol) :param data_type: one or many data_types to add to data_types of input cls. :type data_type: str or Iterable(data_type) """ protocols = cls.get_protocols() data_types = ensure_iterable(cls.__datatype__, iterable=set) if protocol is not None: protocol = ensure_iterable(protocol, iterable=set) protocols |= protocol if data_type is not None: data_type = ensure_iterable(data_type, iterable=set) data_types |= data_type for protocol in protocols: _data_types = Middleware.__MIDDLEWARES__.setdefault(protocol, {}) for datatype in data_types: _data_types[datatype] = cls
def response(data, adapt=True): """Construct a REST response from input data. :param data: data to convert into a REST response. :param kwargs: service function parameters. :param bool adapt: adapt Canopsis data to Ember (default: True) """ # calculate result_data and total related to data type if isinstance(data, tuple): result_data = ensure_iterable(data[0]) total = data[1] else: result_data = None if data is None else ensure_iterable(data) total = 0 if result_data is None else len(result_data) if adapt: # apply transformation for client adapt_canopsis_data_to_ember(result_data) result = { 'total': total, 'data': result_data, 'success': True } return result
def response(data, adapt=True, success=True): """Construct a REST response from input data. :param data: data to convert into a REST response. :param bool adapt: adapt Canopsis data to Ember (default: True) :param bool success: is the response a success (default: True) :rtype: dict """ # calculate result_data and total related to data type if isinstance(data, tuple): result_data = ensure_iterable(data[0]) total = data[1] else: result_data = None if data is None else ensure_iterable(data) total = 0 if result_data is None else len(result_data) if adapt: adapt_canopsis_data_to_ember(result_data) result = {'total': total, 'data': result_data, 'success': success} headers = { 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Pragma': 'no-cache', 'Expires': 0 } for hname in headers: BottleResponse.set_header(hname, headers[hname]) return result
def get_alarms( self, resolved=True, tags=None, exclude_tags=None, timewindow=None ): """ Get alarms from TimedStorage. :param resolved: If ``True``, returns only resolved alarms, else returns only unresolved alarms (default: ``True``). :type resolved: bool :param tags: Tags which must be set on alarm (optional) :type tags: str or list :param exclude_tags: Tags which must not be set on alarm (optional) :type tags: str or list :param timewindow: Time Window used for fetching (optional) :type timewindow: canopsis.timeserie.timewindow.TimeWindow :returns: Iterable of alarms matching """ query = {} if resolved: query['resolved'] = {'$ne': None} else: query['resolved'] = None tags_cond = None if tags is not None: tags_cond = {'$in': ensure_iterable(tags)} notags_cond = None if exclude_tags is not None: notags_cond = {'$nin': ensure_iterable(exclude_tags)} if tags_cond is None and notags_cond is not None: query['tags'] = notags_cond elif tags_cond is not None and notags_cond is None: query['tags'] = tags_cond elif tags_cond is not None and notags_cond is not None: query = {'$and': [query, tags_cond, notags_cond]} return self[Alerts.ALARM_STORAGE].find( _filter=query, timewindow=timewindow )
def get_shared_data(self, shared_ids): """Get all shared data related to input shared ids. :param shared_ids: one or more data. :type shared_ids: list or str :return: depending on input shared_ids:: - one shared id: one list of shared data - list of shared ids: list of list of shared data """ result = [] sids = ensure_iterable(shared_ids, iterable=set) for shared_id in sids: query = {CompositeStorage.SHARED: shared_id} shared_data = self.get_elements(query=query) result.append(shared_data) # return first or data if data is not an iterable if not isiterable(shared_ids, is_str=False): result = get_first(result) return result
def send_events(ws, events, exchange='canopsis.events'): events = ensure_iterable(events) sent_events = [] failed_events = [] retry_events = [] for event in events: try: ws.amqp_pub.canopsis_event(event, exchange) sent_events.append(event) except KeyError as exc: ws.logger.error('bad event: {}'.format(exc)) failed_events.append(event) except AmqpPublishError as exc: ws.logger.error('publish error: {}'.format(exc)) retry_events.append(event) return { 'sent_events': sent_events, 'failed_events': failed_events, 'retry_events': retry_events }
def load_documents(self, data, collection, filename): storage = self.storage.get_backend(collection) data = ensure_iterable(data) for doc in data: if 'loader_id' not in doc: self.logger.error( 'Missing "loader_id" key in document, skipping') self.logger.debug(str(doc)) continue mfilter = {'loader_id': doc['loader_id']} doc_exists = storage.find(mfilter).count() if doc_exists: if not doc.get('loader_no_update', True): storage.update(mfilter, doc, upsert=True) else: self.logger.info(u'Document "{0}" not updatable'.format( doc['loader_id'])) else: storage.update(mfilter, doc, upsert=True)
def update_current_alarm(self, alarm, new_value, tags=None): """ Update alarm's history and tags. :param alarm: Alarm to update :type alarm: dict :param new_value: New history to set on alarm :type new_value: dict :param tags: Tags to add on alarm (optional) :type tags: str or list """ storage = self[Alerts.ALARM_STORAGE] alarm_id = alarm[storage.DATA_ID] alarm_ts = alarm[storage.TIMESTAMP] if tags is not None: for tag in ensure_iterable(tags): if tag not in new_value['tags']: new_value['tags'].append(tag) storage.put(alarm_id, new_value, alarm_ts)
def __init__(self, op, name=None, raw_body=False, payload=None, wsgi_params=None, response=response, adapt=True): """ :param op: ws operation for routing a function :param str name: ws name :param bool raw_body: if True, will set kwargs body to raw request body :param payload: body parameter names (won't be generated in routes) :type payload: str or list of str :param function response: response to apply on decorated function result :param dict wsgi_params: wsgi parameters which will be given to the wsgi such as a keyword :param bool adapt: Adapt Canopsis<->Ember data (default: True) """ super(route, self).__init__() # logger is initialized by WebServer self.logger = logging.getLogger('webserver') self.op = op self.name = name self.raw_body = raw_body self.payload = ensure_iterable(payload) self.response = response self.wsgi_params = wsgi_params self.adapt = adapt self.url = ''
def __init__( self, op, name=None, raw_body=False, payload=None, wsgi_params=None, response=response, adapt=True ): """ :param op: ws operation for routing a function :param str name: ws name :param bool raw_body: if True, will set kwargs body to raw request body :param payload: body parameter names (won't be generated in routes) :type payload: str or list of str :param function response: response to apply on decorated function result :param dict wsgi_params: wsgi parameters which will be given to the wsgi such as a keyword :param bool adapt: Adapt Canopsis<->Ember data (default: True) """ super(route, self).__init__() # logger is initialized by WebServer self.logger = logging.getLogger('webserver') self.op = op self.name = name self.raw_body = raw_body self.payload = ensure_iterable(payload) self.response = response self.wsgi_params = wsgi_params self.adapt = adapt self.url = ''
def get_previous_step(alarm, steptypes, ts=None): """ Get last step in alarm history. :param alarm: Alarm history :type alarm: dict :param steptypes: Step types wanted :type steptypes: str or list :param ts: Timestamp to look from (optional) :type ts: int :returns: Most recent step """ if len(alarm[AlarmField.steps.value]) > 0: if ts is None: ts = alarm[AlarmField.steps.value][-1]['t'] + 1 steptypes = ensure_iterable(steptypes) for step in reversed(alarm[AlarmField.steps.value]): if step['t'] < ts and step['_t'] in steptypes: return step return None
def load_documents(self, data, collection, filename): storage = self.storage.get_backend(collection) data = ensure_iterable(data) for doc in data: if 'loader_id' not in doc: self.logger.error( 'Missing "loader_id" key in document, skipping' ) self.logger.debug(str(doc)) continue mfilter = {'loader_id': doc['loader_id']} doc_exists = storage.find(mfilter).count() if doc_exists: if not doc.get('loader_no_update', True): storage.update(mfilter, doc, upsert=True) else: self.logger.info(u'Document "{0}" not updatable'.format( doc['loader_id'] )) else: storage.update(mfilter, doc, upsert=True)
def update_current_alarm(self, alarm, new_value, tags=None): """ Update alarm's history and tags. :param alarm: Alarm to update :type alarm: dict :param new_value: New history to set on alarm :type new_value: dict :param tags: Tags to add on alarm (optional) :type tags: str or list """ storage = self.alerts_storage alarm_id = alarm[storage.DATA_ID] alarm_ts = alarm[storage.TIMESTAMP] if AlarmField.display_name.value not in new_value: display_name = gen_id() while self.check_if_display_name_exists(display_name): display_name = gen_id() new_value[AlarmField.display_name.value] = display_name if tags is not None: for tag in ensure_iterable(tags): if tag not in new_value[AlarmField.tags.value]: new_value[AlarmField.tags.value].append(tag) storage.put(alarm_id, new_value, alarm_ts) self.watcher_manager.alarm_changed(alarm['data_id'])
def delete(self, names=None): if names is None: names = self.list() names = ensure_iterable(names) for name in names: os.path.remove(self._path(name))
def rest(namespace, ctype, _id=None, body='[]', **kwargs): try: items = ensure_iterable(loads(body)) except ValueError as err: return HTTPError(500, 'Impossible to parse body: {0}'.format(err)) return save_records(ws, namespace, ctype, _id, items)
def rest(namespace, ctype, _id=None, body='[]', **kwargs): try: items = ensure_iterable(json.loads(body)) except ValueError as err: return HTTPError(500, 'Impossible to parse body: {0}'.format(err)) return save_records(ws, namespace, ctype, _id, items)
def _influx_query( self, metric_id, aggregations=[], condition=None, groupby=[] ): query = 'select' if not aggregations: query = '{} value'.format(query) else: for aggr in ensure_iterable(aggregations): query = '{} {}(value),'.format(query, aggr) else: # Remove last ',' query = query[:-1] query = '{} from {}'.format(query, metric_id) if condition is not None: query = '{} where {}'.format(query, condition) if groupby: query = '{} group by'.format(query) for gb in ensure_iterable(groupby): query = '{} {},'.format(query, gb) # Remove last ',' else: query = query[:-1] self.logger.debug(u'Processing query `{}`'.format(query)) result = self.influxdbstg.raw_query(query) # If something went wrong (bad metric_id, bad aggregation...), # InfluxDBStorage will just return None if result is None: raise ValueError( 'Query `{}` failed : unable to retrieve stats'.format(query) ) return result
def get(self, names, version=-1): names = ensure_iterable(names) result = [] for name in names: gridout = self.gridfs.get_version(filename=name, version=version) fs = MongoFileStream(gridout) result.append(fs) return result
def unshare_data(self, data, cache=False): """Remove share property from input data. :param data: one or more data to unshare. :param bool cache: use query cache if True (False by default). """ data = ensure_iterable(data) for d in data: if CompositeStorage.SHARED in d: d[CompositeStorage.SHARED] = str(uuid()) path, name = self.get_path_with_name(d) self.put(path=path, name=name, data=d, cache=cache)
def response(data, adapt=True, success=True): """Construct a REST response from input data. :param data: data to convert into a REST response. :param bool adapt: adapt Canopsis data to Ember (default: True) :param bool success: is the response a success (default: True) :rtype: dict """ # calculate result_data and total related to data type if isinstance(data, tuple): result_data = ensure_iterable(data[0]) total = data[1] else: result_data = None if data is None else ensure_iterable(data) total = 0 if result_data is None else len(result_data) if adapt: adapt_canopsis_data_to_ember(result_data) result = { 'total': total, 'data': result_data, 'success': success } headers = { 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Pragma': 'no-cache', 'Expires': 0 } for hname in headers: BottleResponse.set_header(hname, headers[hname]) return result
def delete(self, names=None): if names is None: names = self.gridfs.list() names = ensure_iterable(names) for name in names: while True: fs = self.get(name) if fs is None: break self.gridfs.delete(file_id=fs.get_inner_object()._id)
def send_event(event, url=None): if ws.enable_crossdomain_send_events and url is not None: payload = { 'event': json.dumps(event) } response = requests.post(url, data=payload) if response.status_code != 200: api_response = json.loads(response.text) return (api_response['data'], api_response['total']) else: return HTTPError(response.status_code, response.text) else: events = ensure_iterable(event) exchange = ws.amqp.exchange_name_events for event in events: if schema.validate(event, 'cevent'): sname = 'cevent.{0}'.format(event['event_type']) if schema.validate(event, sname): if event['event_type'] == 'eue': sname = 'cevent.eue.{0}'.format( event['type_message'] ) if not schema.validate(event, sname): continue rk = '{0}.{1}.{2}.{3}.{4}'.format( u'{0}'.format(event['connector']), u'{0}'.format(event['connector_name']), u'{0}'.format(event['event_type']), u'{0}'.format(event['source_type']), u'{0}'.format(event['component']) ) if event['source_type'] == 'resource': rk = '{0}.{1}'.format(rk, event['resource']) ws.amqp.publish(event, rk, exchange) return events
def put_elts(self, elts, graph_ids=None, cache=False): """Put graph elements in DB. :param elts: graph elements to put in DB. :type elts: dict, GraphElement or list of dict/GraphElement. :param str graph_ids: element graph id. None if elt is a graph. :param bool cache: use query cache if True (False by default). :return: corresponding graph elts. :rtype: list of GraphElements """ result= [] # ensure elts is a list if isinstance(elts, (dict, GraphElement)): elts= [elts] for elt in elts: gelt= elt # in case of dict, get the corresponding graph elt and save it if isinstance(gelt, dict): gelt= GraphElement.new(**gelt) gelt.process(event={}) gelt.save(manager=self, cache=cache) else: # in case of graphelt, save its serialized form in db serialized_elt= gelt.to_dict() # put elt value in storage self[GraphManager.STORAGE].put_element( _id=elt.id, element=serialized_elt, cache=cache ) # add the graphical element to the result result.append(gelt) # associate all elt ids with all graph ids if graph_ids is not None: # eliminate doublons of elts elt_ids= set([gelt.id for gelt in result]) # ensure graph_ids is a basestring graph_ids= ensure_iterable(graph_ids) graphs= self[GraphManager.STORAGE].get_elements(ids=graph_ids) # add elt ids in elts of graphs for graph in graphs: graph[Graph.ELTS]= list(graph[Graph.ELTS] | elt_ids) # save all graphs self[GraphManager.STORAGE].put_elements(elements=graphs) return result
def get_protocols(cls): """Get all protocols declared in the class hierarchy of cls :return: set of protocols registered in the class tree of input cls :rtype: set([str]) """ protocols = ensure_iterable(cls.__protocol__, iterable=set) for base_cls in cls.__bases__: if issubclass(base_cls, Middleware): base_protocols = base_cls.get_protocols() protocols |= base_protocols result = protocols return result
def send_event(event, url=None): if ws.enable_crossdomain_send_events and url is not None: payload = {"event": json.dumps(event)} response = requests.post(url, data=payload) if response.status_code != 200: api_response = json.loads(response.text) return (api_response["data"], api_response["total"]) else: return HTTPError(response.status_code, response.text) else: events = ensure_iterable(event) exchange = ws.amqp.exchange_name_events for event in events: if schema.validate(event, "cevent"): sname = "cevent.{0}".format(event["event_type"]) if schema.validate(event, sname): if event["event_type"] == "eue": sname = "cevent.eue.{0}".format(event["type_message"]) if not schema.validate(event, sname): continue rk = "{0}.{1}.{2}.{3}.{4}".format( event["connector"], event["connector_name"], event["event_type"], event["source_type"], event["component"], ) if event["source_type"] == "resource": rk = "{0}.{1}".format(rk, event["resource"]) ws.amqp.publish(event, rk, exchange) return events
def load(self, path): try: loaded = [] for fpath in os.listdir(path): if fpath.endswith('.json'): fullpath = os.path.join(path, fpath) with open(fullpath) as f: data = ensure_iterable(json.load(f)) loaded += data except Exception as err: self.logger.error(u'Unable to load JSON files "{0}": {1}'.format( path, err)) loaded = [] return loaded
def find(entity_ids=None, behaviors=None, start=None, end=None): """Find documents related to input entity id(s) and behavior(s). :param entity_ids: :type entity_ids: list or str :param behaviors: :type behaviors: list or str :param int start: start timestamp. :param int end: end timestamp. :return: entity documents with input behaviors. :rtype: list """ query = PBehaviorManager.get_query(behaviors) entity_ids = ensure_iterable(entity_ids) result = pbm.values( sources=entity_ids, query=query, dtstart=start, dtend=end ) return result
def find( self, names=None, meta=None, sort=None, limit=-1, skip=0, with_meta=False ): if names is not None: names = ensure_iterable(names) result = [] for _, _, files in os.walk(self.uri): for filename in files: # ignore meta files if not filename.endswith(UnixFileStorage.META_EXT): if names is not None and filename not in names: continue if meta is not None: metadata = self.get_meta(filename) if not check(meta, metadata): continue result.append(filename) result = result[skip:limit] result = [open(filename) for filename in result] if with_meta: result = [(f, self.get_meta(f.name)) for f in result] if sort is not None: raise NotImplementedError('sort is not yet supported') return result
def load(self, path): try: loaded = [] for fpath in os.listdir(path): if fpath.endswith('.json'): fullpath = os.path.join(path, fpath) with open(fullpath) as f: data = ensure_iterable(json.load(f)) loaded += data except Exception as err: self.logger.error(u'Unable to load JSON files "{0}": {1}'.format( path, err )) loaded = [] return loaded
def send_events(ws, am, events, exchange='canopsis.events'): events = ensure_iterable(events) sent_events = [] failed_events = [] retry_events = [] for event in events: if not is_valid(ws, event): ws.logger.error("event {}/{} is invalid".format( event.get("resource"), event.get("component"))) failed_events.append(event) continue try: transformed_event = transform_event(ws, am, event) except Exception as e: ws.logger.error('Failed to transform event : {}'.format(e)) failed_events.append(event) continue try: ws.amqp_pub.canopsis_event(transformed_event, exchange) sent_events.append(transformed_event) except KeyError as exc: ws.logger.error('bad event: {}'.format(exc)) failed_events.append(transformed_event) except AmqpPublishError as exc: ws.logger.error('publish error: {}'.format(exc)) retry_events.append(transformed_event) return { 'sent_events': sent_events, 'failed_events': failed_events, 'retry_events': retry_events }
def get_alarms( self, resolved=True, tags=None, exclude_tags=None, timewindow=None, snoozed=False ): """ Get alarms from TimedStorage. :param resolved: If ``True``, returns only resolved alarms, else returns only unresolved alarms (default: ``True``). :type resolved: bool :param tags: Tags which must be set on alarm (optional) :type tags: str or list :param exclude_tags: Tags which must not be set on alarm (optional) :type tags: str or list :param timewindow: Time Window used for fetching (optional) :type timewindow: canopsis.timeserie.timewindow.TimeWindow :param snoozed: If ``False``, return all non-snoozed alarms, else returns alarms even if they are snoozed. :type snoozed: bool :returns: Iterable of alarms matching: {alarm_id: [alarm_dict]} """ query = {} if resolved: query['resolved'] = {'$ne': None} else: query['resolved'] = None tags_cond = None if tags is not None: tags_cond = {'$all': ensure_iterable(tags)} notags_cond = None if exclude_tags is not None: notags_cond = {'$not': {'$all': ensure_iterable(exclude_tags)}} if tags_cond is None and notags_cond is not None: query['tags'] = notags_cond elif tags_cond is not None and notags_cond is None: query['tags'] = tags_cond elif tags_cond is not None and notags_cond is not None: query = {'$and': [ query, {'tags': tags_cond}, {'tags': notags_cond} ]} # used to fetch alarms that were never snoozed OR alarms for which the snooze has expired if not snoozed: no_snooze_cond = { '$or': [ {AlarmField.snooze.value: None}, {'snooze.val': {'$lte': int(time())}} ] } query = {'$and': [query, no_snooze_cond]} alarms_by_entity = self.alerts_storage.find( _filter=query, timewindow=timewindow ) for entity_id, alarms in alarms_by_entity.items(): entity = self.context_manager.get_entities_by_id(entity_id) try: entity = entity[0] except IndexError: entity = {} entity['entity_id'] = entity_id for alarm in alarms: alarm['entity'] = entity return alarms_by_entity
class Formulas(object): """Class that reads formulas and parse it using EBNF grammar""" # map operator symbols to corresponding arithmetic operations global epsilon epsilon = 1e-12 opn = { "+": operator.add, "-": operator.sub, "*": operator.mul, "/": operator.truediv, "^": operator.pow } fn = { "sin": math.sin, "cos": math.cos, "tan": math.tan, "abs": abs, "trunc": lambda a: int(a), "round": round, "max": lambda l: max(float(i) for i in l), "min": lambda l: min(float(i) for i in ensure_iterable(l)), "sum": lambda l: sum(float(i) for i in l), "sgn": lambda a: abs(a) > epsilon and ((a > 0) - (a < 0)) or 0 } def __init__(self, _dict=None): self.exprStack = [] self._bnf = None self._dict = _dict # The dictionnary value as dictionnary {'x':2} self.variables = _dict def push_first(self, strg, loc, toks): ''' Define an action to apply on the matched tokens :param strg: is the original parse string :param loc: is the location in the string where matching started :param toks: is the list of the matched tokens ''' self.exprStack.append(toks[0]) def push_minus(self, strg, loc, toks): ''' Define an action to apply on the matched tokens :param strg: is the original parse string. :param loc: is the location in the string where matching started. :param toks: is the list of the matched tokens. ''' if toks and toks[0] == '-': self.exprStack.append('unary -') def _import(self, _dict): ''' set variables data. :param _dict: variables and thier values. ''' self._dict = _dict def reset(self): ''' Reset the variables and thier values. ''' self._dict = {} def bnf(self): ''' The BNF grammar is defined bellow. expop :: '^' multop :: '*' | '/' addop :: '+' | '-' integer :: ['+' | '-'] '0'..'9'+ atom :: PI | E | real | fn '(' expr ')' | '(' expr ')' factor :: atom [ expop factor ]* term :: factor [ multop factor ]* expr :: term [ addop term ]* ''' if not self._bnf: point = Literal(".") e = CaselessLiteral("E") fnumber = Combine( Word("+-" + nums, nums) + Optional(point + Optional(Word(nums))) + Optional(e + Word("+-" + nums, nums))) ident = Word(alphas, alphas + nums + "_$") minus = Literal("-") plus = Literal("+") div = Literal("/") mult = Literal("*") rpar = Literal(")").suppress() lpar = Literal("(").suppress() addop = plus | minus multop = mult | div expop = Literal("^") pi = CaselessLiteral("PI") expr = Forward() atom = (Optional("-") + (pi | e | fnumber | ident + lpar + delimitedList(expr) + rpar).setParseAction(self.push_first) | (lpar + expr.suppress() + rpar)).setParseAction( self.push_minus) # The right way to define exponentiation is -> 2^3^2 = 2^(3^2), # not (2^3)^2. factor = Forward() factor << atom + ZeroOrMore( (expop + factor).setParseAction(self.push_first)) term = factor + ZeroOrMore( (multop + factor).setParseAction(self.push_first)) expr << term + ZeroOrMore( (addop + term).setParseAction(self.push_first)) self._bnf = expr return self._bnf def evaluate_parsing(self, parsing_result): ''' ''' op = parsing_result.pop() if op == 'unary -': return -self.evaluate_parsing(parsing_result) if op in "+-*/^": op2 = self.evaluate_parsing(parsing_result) op1 = self.evaluate_parsing(parsing_result) return self.opn[op](op1, op2) elif op.lower() == "pi": return math.pi # 3.1415926535 elif op.lower() == "e": return math.e # 2.718281828 elif op.lower() in self.fn: t_op = op.lower() if t_op in ('max', 'min', 'sum'): if type(parsing_result) is list: return self.fn[t_op](parsing_result) return self.fn[t_op](self.evaluate_parsing(parsing_result)) return self.fn[op](self.evaluate_parsing(parsing_result)) elif re.search('^[a-zA-Z][a-zA-Z0-9_]*$', op): if op in self._dict: return self._dict[op] else: return 0 elif op[0].isalpha(): return 0 else: return float(op) def evaluate(self, formula): ''' Evaluate the formula ''' if self._dict is not None: for k, v in self._dict.iteritems(): formula = formula.replace(str(k), str(v)) self.exprStack = [] # reset the stack before each eval try: results = self.bnf().parseString(formula) except ParseException: results = ['Parse Failure', formula] if len(results) == 0 or results[0] == 'Parse Failure': return 'Parse Failure-{}'.format(formula) val = self.evaluate_parsing(self.exprStack[:]) return val
def delete(self, names): names = ensure_iterable(names) for name in names: self.gridfs.delete(file_id=name)
def get_alarms(self, resolved=True, tags=None, exclude_tags=None, timewindow=None, snoozed=False): """ Get alarms from TimedStorage. :param resolved: If ``True``, returns only resolved alarms, else returns only unresolved alarms (default: ``True``). :type resolved: bool :param tags: Tags which must be set on alarm (optional) :type tags: str or list :param exclude_tags: Tags which must not be set on alarm (optional) :type tags: str or list :param timewindow: Time Window used for fetching (optional) :type timewindow: canopsis.timeserie.timewindow.TimeWindow :param snoozed: If ``False``, return all non-snoozed alarms, else returns alarms even if they are snoozed. :type snoozed: bool :returns: Iterable of alarms matching: {alarm_id: [alarm_dict]} """ query = {} if resolved: query['resolved'] = {'$ne': None} else: query['resolved'] = None tags_cond = None if tags is not None: tags_cond = {'$all': ensure_iterable(tags)} notags_cond = None if exclude_tags is not None: notags_cond = {'$not': {'$all': ensure_iterable(exclude_tags)}} if tags_cond is None and notags_cond is not None: query['tags'] = notags_cond elif tags_cond is not None and notags_cond is None: query['tags'] = tags_cond elif tags_cond is not None and notags_cond is not None: query = { '$and': [query, { 'tags': tags_cond }, { 'tags': notags_cond }] } # used to fetch alarms that were never snoozed OR alarms for which the snooze has expired if not snoozed: no_snooze_cond = { '$or': [{ AlarmField.snooze.value: None }, { 'snooze.val': { '$lte': int(time()) } }] } query = {'$and': [query, no_snooze_cond]} alarms_by_entity = self.alerts_storage.find(_filter=query, timewindow=timewindow) for entity_id, alarms in alarms_by_entity.items(): entity = self.context_manager.get_entities_by_id(entity_id) try: entity = entity[0] except IndexError: entity = {} entity['entity_id'] = entity_id for alarm in alarms: alarm['entity'] = entity return alarms_by_entity
def get_alarms( self, resolved=True, tags=None, exclude_tags=None, timewindow=None, snoozed=False ): """ Get alarms from TimedStorage. :param resolved: If ``True``, returns only resolved alarms, else returns only unresolved alarms (default: ``True``). :type resolved: bool :param tags: Tags which must be set on alarm (optional) :type tags: str or list :param exclude_tags: Tags which must not be set on alarm (optional) :type tags: str or list :param timewindow: Time Window used for fetching (optional) :type timewindow: canopsis.timeserie.timewindow.TimeWindow :param snoozed: If ``False``, return all non-snoozed alarms, else returns alarms even if they are snoozed. :type snoozed: bool :returns: Iterable of alarms matching """ query = {} if resolved: query['resolved'] = {'$ne': None} else: query['resolved'] = None tags_cond = None if tags is not None: tags_cond = {'$all': ensure_iterable(tags)} notags_cond = None if exclude_tags is not None: notags_cond = {'$not': {'$all': ensure_iterable(exclude_tags)}} if tags_cond is None and notags_cond is not None: query['tags'] = notags_cond elif tags_cond is not None and notags_cond is None: query['tags'] = tags_cond elif tags_cond is not None and notags_cond is not None: query = {'$and': [ query, {'tags': tags_cond}, {'tags': notags_cond} ]} if not snoozed: no_snooze_cond = {'$or': [ {'snooze': None}, {'snooze.val': {'$lte': int(time())}} ] } query = {'$and': [query, no_snooze_cond]} alarms_by_entity = self[Alerts.ALARM_STORAGE].find( _filter=query, timewindow=timewindow ) cm = Context() for entity_id, alarms in alarms_by_entity.items(): entity = cm.get_entity_by_id(entity_id) entity['entity_id'] = entity_id for alarm in alarms: alarm['entity'] = entity return alarms_by_entity
def get_records(ws, namespace, ctype=None, _id=None, **params): options = { "limit": 20, "start": 0, "search": None, "filter": None, "sort": None, "query": None, "onlyWritable": False, "noInternal": False, "ids": [], "multi": None, "fields": {}, } for key in options.keys(): options[key] = params.get(key, options[key]) # Ensure sort always evaluates to list sort = options["sort"] if not sort: sort = [] else: sort = ensure_iterable(sort) if isinstance(sort, basestring): # NOQA try: sort = loads(sort) except ValueError as json_error: ws.logger.warning("Unable to parse sort field : {} {}".format(sort, json_error)) sort = [] # Generate MongoDB sorting query msort = [ (item["property"], 1 if item["direction"] == "DESC" else -1) for item in sort if item.get("property", None) is not None ] # Generate MongoDB filter mfilter = {} if isinstance(options["filter"], list): for item in options["filter"]: mfilter[item["property"]] = item["value"] elif isinstance(options["filter"], dict): mfilter = options["filter"] if options["multi"]: mfilter["crecord_type"] = {"$in": options["multi"].split(",")} elif ctype: mfilter["crecord_type"] = ctype if options["query"]: # FIXME: bad query can't be indexed mfilter["crecord_name"] = {"$regex": ".*{0}.*".format(options["query"]), "$options": "i"} if options["search"]: # FIXME: bad query can't be indexed mfilter["_id"] = {"$regex": ".*{0}.*".format(options["search"]), "$options": "i"} ids = options["ids"] if not _id else _id.split(",") # Perform query total = 0 records = [] if len(ids) > 0: try: records = ws.db.get(ids, namespace=namespace) except KeyError: records = [] if isinstance(records, Record): records = [records] total = 1 elif isinstance(records, list): total = len(records) else: total = 0 if total == 0: return HTTPError(404, "IDs not found: {0}".format(ids)) else: records, total = ws.db.find( mfilter, sort=msort, limit=options["limit"], offset=options["start"], with_total=True, namespace=namespace ) # Generate output output = [] noInternal = options["noInternal"] for record in records: if record: # TODO: make use of onlyWritable # This can be done with canopsis.old.account, but the goal is to # use the new permissions/rights system to do it. dump = record.data.get("internal", False) if noInternal else True if dump: data = record.dump(json=True) data["id"] = data["_id"] if "next_run_time" in data: data["next_run_time"] = str(data["next_run_time"]) # TODO: Handle projection in ws.db.find() if options["fields"]: for item in data.keys(): if item not in options["fields"]: del data[item] output.append(data) with open("/tmp/lolog", "a+") as fh: fh.write("output={},total={}\n".format(output, total)) return output, total
def get_records(ws, namespace, ctype=None, _id=None, **params): options = { 'limit': 20, 'start': 0, 'search': None, 'filter': None, 'sort': None, 'query': None, 'onlyWritable': False, 'noInternal': False, 'ids': [], 'multi': None, 'fields': {} } for key in options.keys(): options[key] = params.get(key, options[key]) # Ensure sort always evaluates to list sort = options['sort'] if not sort: sort = [] else: sort = ensure_iterable(sort) if isinstance(sort, basestring): try: sort = json.loads(sort) except ValueError as json_error: ws.logger.warning('Unable to parse sort field : {} {}'.format( sort, json_error )) sort = [] # Generate MongoDB sorting query msort = [ ( item['property'], 1 if item['direction'] == 'DESC' else -1 ) for item in sort if item.get('property', None) is not None ] # Generate MongoDB filter mfilter = {} if isinstance(options['filter'], list): for item in options['filter']: mfilter[item['property']] = item['value'] elif isinstance(options['filter'], dict): mfilter = options['filter'] if options['multi']: mfilter['crecord_type'] = { '$in': options['multi'].split(',') } elif ctype: mfilter['crecord_type'] = ctype if options['query']: # FIXME: bad query can't be indexed mfilter['crecord_name'] = { '$regex': '.*{0}.*'.format(options['query']), '$options': 'i' } if options['search']: # FIXME: bad query can't be indexed mfilter['_id'] = { '$regex': '.*{0}.*'.format(options['search']), '$options': 'i' } ids = options['ids'] if not _id else _id.split(',') # Perform query total = 0 records = [] if len(ids) > 0: try: records = ws.db.get(ids, namespace=namespace) except KeyError: records = [] if isinstance(records, Record): records = [records] total = 1 elif isinstance(records, list): total = len(records) else: total = 0 if total == 0: return HTTPError(404, 'IDs not found: {0}'.format(ids)) else: records, total = ws.db.find( mfilter, sort=msort, limit=options['limit'], offset=options['start'], with_total=True, namespace=namespace ) # Generate output output = [] noInternal = options['noInternal'] for record in records: if record: # TODO: make use of onlyWritable # This can be done with canopsis.old.account, but the goal is to # use the new permissions/rights system to do it. dump = record.data.get('internal', False) if noInternal else True if dump: data = record.dump(json=True) data['id'] = data['_id'] if 'next_run_time' in data: data['next_run_time'] = str(data['next_run_time']) # TODO: Handle projection in ws.db.find() if options['fields']: for item in data.keys(): if item not in options['fields']: del data[item] output.append(data) return output, total
def get_records(ws, namespace, ctype=None, _id=None, **params): options = { 'limit': 20, 'start': 0, 'search': None, 'filter': None, 'sort': None, 'query': None, 'onlyWritable': False, 'noInternal': False, 'ids': [], 'multi': None, 'fields': {} } for key in options.keys(): options[key] = params.get(key, options[key]) # Ensure sort always evaluates to list sort = options['sort'] if not sort: sort = [] else: sort = ensure_iterable(sort) # Generate MongoDB sorting query msort = [(item['property'], 1 if item['direction'] == 'DESC' else -1) for item in sort if item.get('property', None) is not None] # Generate MongoDB filter mfilter = {} if isinstance(options['filter'], list): for item in options['filter']: mfilter[item['property']] = item['value'] elif isinstance(options['filter'], dict): mfilter = options['filter'] if options['multi']: mfilter['crecord_type'] = {'$in': options['multi'].split(',')} elif ctype: mfilter['crecord_type'] = ctype if options['query']: # FIXME: bad query can't be indexed mfilter['crecord_name'] = { '$regex': '.*{0}.*'.format(options['query']), '$options': 'i' } if options['search']: # FIXME: bad query can't be indexed mfilter['_id'] = { '$regex': '.*{0}.*'.format(options['search']), '$options': 'i' } ids = options['ids'] if not _id else _id.split(',') # Perform query total = 0 records = [] if len(ids) > 0: try: records = ws.db.get(ids, namespace=namespace) except KeyError: records = [] if isinstance(records, Record): records = [records] total = 1 elif isinstance(records, list): total = len(records) # TODO: ne peut pas fonctionner if total == 0: return HTTPError(404, 'IDs not found: {0}'.format(ids)) else: records, total = ws.db.find(mfilter, sort=msort, limit=options['limit'], offset=options['start'], with_total=True, namespace=namespace) # Generate output output = [] noInternal = options['noInternal'] for record in records: if record: # TODO: make use of onlyWritable # This can be done with canopsis.old.account, but the goal is to # use the new permissions/rights system to do it. dump = record.data.get('internal', False) if noInternal else True if dump: data = record.dump(json=True) data['id'] = data['_id'] if 'next_run_time' in data: data['next_run_time'] = str(data['next_run_time']) # TODO: Handle projection in ws.db.find() if options['fields']: for item in data.keys(): if item not in options['fields']: del data[item] output.append(data) return output, total