def __init__( self, config ): self._parser_lock = Lock() self.qp = parsing.Lr( query_parser.spec ) self.sp = parsing.Lr( selector_parser.spec ) host = config["host"] port = config["port"] user = config["user"] dbname = config["database"] passwd = config["password"] self.store = MongoDBMetricStore(host_name=host, port=port, db_name=dbname, username=user, password=passwd) #self.store = None factory = {} def _search_subclasses( base ): for c in base.__subclasses__(): factory[c.__name__] = c _search_subclasses( c ) _search_subclasses( ContainerObject ) self.factory = factory
class HierarchyAdapterMonitorMongoDB( HierarchyAdapter ): def __init__( self, config ): self._parser_lock = Lock() self.qp = parsing.Lr( query_parser.spec ) self.sp = parsing.Lr( selector_parser.spec ) host = config["host"] port = config["port"] user = config["user"] dbname = config["database"] passwd = config["password"] self.store = MongoDBMetricStore(host_name=host, port=port, db_name=dbname, username=user, password=passwd) #self.store = None factory = {} def _search_subclasses( base ): for c in base.__subclasses__(): factory[c.__name__] = c _search_subclasses( c ) _search_subclasses( ContainerObject ) self.factory = factory def oid_to_rid(self, oid): return ObjectId(oid) def rid_to_oid(self, rid): return str(rid) def cleanup_content_get( self, content ): clean_content = content.copy() clean_content["_id"] = self.rid_to_oid( content["_id"] ) return clean_content def _parse_query( self, arg ): with self._parser_lock: self.qp.reset() stream = StringIO( arg ) lexer = QueryLexer( self.qp, stream ) return lexer.parse().value def _parse_selector( self, arg ): with self._parser_lock: self.sp.reset() stream = StringIO( arg ) lexer = QueryLexer( self.sp, stream ) return lexer.parse().value def client( self, op, *args ): if op == "describe": if len(args) == 0: return {} return self.op_describe( args[0] ) elif op == "select": if len(args) == 0: return [] query = self._parse_query( args[0] ) if len(args) > 1: selector = self._parse_selector( args[1] ) else: selector = None return self.op_select( query, selector ) elif op == "locate": if len(args) == 0: return [] query = self._parse_query( args[0] ) return self.op_locate( query ) if op == "menu_types": return self.op_menu_types( ) def op_describe( self, key ): """ Return object's class metadata information. """ if key in self.factory: # class name was provided cls = self.factory[key] else: # try locating this obj_id try: obj = self.store.find_md( {"_id": self.oid_to_rid(key)} )[0] except Exception as e: print "DESCRIBE: locating _id=%s failed. %r" % (key, e) #log.warning("DESCRIBE: locating _id=%s failed. %r" % (key, e)) return False cls = self.factory[obj["_type"]] result = {} for var in ["ATTRIBUTES", "CONTAINS_TYPES", "KEY_ATTRIBUTE"]: if hasattr( cls, var ): result[var] = _repl_func_by_name( getattr( cls, var ) ) # is this needed? result["_type"] = cls.__name__ return result def op_menu_types( self ): return [ "MGroup", "MHost", "MMetric" ] def op_locate( self, query ): selector = self._parse_selector( "[_id]" ) return [ self._process_selector( el, selector )["_id"] for el in self._do_query( query ) ] def op_select( self, query, selector ): if selector is None: return [ self.cleanup_content_get( el ) for el in self._do_query( query ) ] else: return [ self._process_selector( el, selector ) for el in self._do_query( query ) ] def _process_selector( self, obj, selector, strictly_selected=False ): clname = obj["_type"] res = {} if not strictly_selected: # following attributes are always included, except when 'strictly_selected' is True res = { "_id": self.rid_to_oid( obj["_id"] ), "_type": clname } c = self.factory[clname] clmd = c.ATTRIBUTES for attr in selector: a = attr.name if a in res: continue elif a == "_id": res[a] = self.rid_to_oid( obj["_id"] ) continue value = obj.get( a, None ) if value is not None: res[a] = value else: attrmd = clmd.get( a, None ) if attrmd is not None: func = attrmd.get( "function", None ) if func is not None: args = [] if attr.arguments is not None: args = [ str(arg.value) for arg in attr.arguments ] res[a] = func( self, self.rid_to_oid(obj["_id"]), obj, *args ) else: res[a] = None return res def _do_query( self, query ): ( id_list, path ) = query #print "_do_query: id_list=%r, path=%r" % (id_list, path) objs = [] hpaths = [] find = None if id_list is not None: if isinstance( id_list, query_parser.OIDList ): find = {"_id": {"$in": [self.oid_to_rid(i.value) for i in id_list.value]}} else: # This is a list of hpaths or query expressions for q in id_list.value: objs.extend( self._do_query( q.value ) ) if path is None: return objs else: # TODO: or not? # When does this case occur? hpaths_list = [ self.oid_to_rid(obj["hpath"]) for obj in objs ] find = {"hpath": {"$in": hpaths_list}} if find is not None: if path is None: objs = self.store.find_md( find ) else: # TODO: don't use collection here... but not so urgent # WHAT IS THIS NEEDED FOR? print "DISTINCT called!? path=%r, find=%r" % (path, find) hpaths = self.store._col_md.distinct("hpath", query=find) ## ## internal functions ## def append_hpath(hpaths, data): #print "append_hpath start: data=%r, hpaths=%r" % (data, hpaths) if len(hpaths) == 0: hpaths.append(data) else: for i in xrange(len(hpaths)): hpaths[i] += data #print "append_hpath end: hpaths=%r" % hpaths return hpaths def strip_hpath(hpaths, levels=1): """Strip 'levels' hpath elements on the right of each of the hpaths passed as argument. I.e. return the directory of the hpaths located 'levels' up in the tree. """ #print "strip_hpath start: hpaths=%r" % hpaths for i in xrange(len(hpaths)): if hpaths[i].startswith("/"): hpath = hpaths[i][1:] else: hpath = hpaths[i] elems = hpath.split("/") hpaths[i] = "/" + "/".join(elems[:len(elems) - levels]) #print "strip_hpath end: hpaths=%r" % hpaths return hpaths def expand_tripledot(hpaths): """Tripledot "..." expands to all parent paths up to root. """ new_hpaths = [] for hpath in hpaths: new_hpaths.append(hpath) while hpath != "/": hpath = strip_hpath([hpath])[0] new_hpaths.append(hpath) return new_hpaths def resolve_hpaths(hpaths, has_regexp, condition, full=False): #print "resolve_hpaths start:\nhpaths=%r\nhas_regexp=%r\ncondition=%r\nfull=%r" % \ # (hpaths, has_regexp, condition, full) objs = [] proj = None if not full: proj = {"hpath": True} if not has_regexp: if condition: objs = self.store.find_md({"$and": [condition, {"hpath": {"$in": hpaths}}]}, projection=proj) else: objs = self.store.find_md({"hpath": {"$in": hpaths}}, projection=proj) else: ors = [] for hpath in hpaths: ors.append({"hpath": {"$regex": "^" + hpath + "$"}}) if condition: objs = self.store.find_md({"$and": [condition, {"$or": ors}]}, projection=proj) else: objs = self.store.find_md({"$or": ors}, projection=proj) #print "resolve_hpaths end: _Cursor__spec: %r" % objs._Cursor__spec if objs.count() == 0: return None if not full: return [obj["hpath"] for obj in objs] return [obj for obj in objs] has_regexp = False arbitrary_depth = False cond = {} if path is not None: #for el in path.value: for i in xrange(len(path.value)): el = path.value[i] #print "el = ", el #pdb.set_trace() if not el.is_filter: if el.name is None: # / encountered if len(path.value) == 1: objs = [{"hpath": "/", "_type": "MRoot", "_id": 0}] else: # // encountered if i == len(path.value) - 1: hpaths = ["/"] arbitrary_depth = True elif isinstance( el.name, tokens.Dot ): arbitrary_depth = False # .. found elif isinstance( el.name, tokens.DoubleDot ): if arbitrary_depth: # //.. raise NotImplementedError if has_regexp: hpaths = resolve_hpaths(hpaths, has_regexp, cond, full=False) has_regexp = False hpaths = strip_hpath(hpaths) arbitrary_depth = False elif isinstance( el.name, tokens.TripleDot ): hpaths = expand_tripledot(hpaths) arbitrary_depth = False elif isinstance( el.name, tokens.Asterisk ): if arbitrary_depth: hpaths = append_hpath(hpaths, "/.*" ) else: hpaths = append_hpath(hpaths, "/[^/]+" ) arbitrary_depth = False has_regexp = True # force resolving else: # element it is a string if el.name in self.factory: assert False, "Query for class %s without '[]'" % el.name if arbitrary_depth: hpaths = append_hpath(hpaths, "/.*" ) has_regexp = True hpaths = append_hpath(hpaths, "/" + el.name ) arbitrary_depth = False else: # # Class[condition] # if el.condition is None: cond = {} else: cond = get_find_condition( el.condition ) #print "condition = %r" % cond if isinstance( el.name, tokens.Dot ): pass elif isinstance( el.name, tokens.DoubleDot ): if arbitrary_depth: raise NotImplementedError if has_regexp: hpaths = resolve_hpaths(hpaths, has_regexp, cond, full=False) has_regexp = False hpaths = strip_hpath(hpaths) if cond: has_regexp = True arbitrary_depth = False elif isinstance( el.name, tokens.TripleDot ): hpaths = expand_tripledot(hpaths) arbitrary_depth = False has_regexp = True # force resolving the condition elif isinstance( el.name, tokens.Asterisk ): if arbitrary_depth: hpaths = append_hpath(hpaths, "/.*" ) else: hpaths = append_hpath(hpaths, "/[^/]+" ) arbitrary_depth = False has_regexp = True # force resolving the condition else: # TODO: add Class match if el.name not in self.factory: assert False, "ERROR: match for unsupported object type '%s'!" % el.name if arbitrary_depth: hpaths = append_hpath(hpaths, "/.*" ) else: hpaths = append_hpath(hpaths, "/[^/]+" ) has_regexp = True if cond: cond = {"$and": [cond, {"_type": el.name}]} else: cond = {"_type": el.name} arbitrary_depth = False if has_regexp: # # and now get the results! # if i == len(path.value) - 1: objs = resolve_hpaths(hpaths, has_regexp, cond, full=True) break else: hpaths = resolve_hpaths(hpaths, has_regexp, cond, full=False) if hpaths is None: break has_regexp = False if objs is None or hpaths is None: return [] if len(objs) == 0 and len(hpaths) > 0: objs = resolve_hpaths(hpaths, has_regexp, {}, full=True) #pdb.set_trace() if not isinstance(objs, list): objs = [obj for obj in objs] return objs