def get_path(self): paths = [] for cp in config.get_customized_paths(self.PREFIX, prefer_custom=True): path = os.path.join(cp, self.name) if os.path.exists(path): paths += [path] return paths
def get_loader(self, name): """ Load loader and return BaseLoader instance. Returns None when no loader found or loading error occured """ with self.lock: loader = self.loaders.get(name) if not loader: logger.info("Loading loader %s", name) for p in config.get_customized_paths("", prefer_custom=True): base = "noc.custom" if p else "noc.core" module_name = "%s.etl.loader.%s" % (base, name) try: sm = __import__(module_name, {}, {}, "*") for n in dir(sm): o = getattr(sm, n) if ( inspect.isclass(o) and issubclass(o, BaseLoader) and o.__module__ == sm.__name__ ): loader = o break logger.error("Loader not found: %s", name) except Exception as e: logger.error("Failed to load loader %s: %s", name, e) loader = None if loader: break self.loaders[name] = loader return loader
def get_model(self, name): with self.lock: model = self.models.get(name) if not model: self.logger.info("Loading loader %s", name) if not self.is_valid_name(name): self.logger.error("Invalid loader name: %s", name) return None for p in config.get_customized_paths("", prefer_custom=True): path = os.path.join(p, "bi", "models", "%s.py" % name) if not os.path.exists(path): continue if p: # Customized model base_name = os.path.basename(os.path.dirname(p)) module_name = "%s.bi.models.%s" % (base_name, name) else: # Common model module_name = "noc.bi.models.%s" % name model = self.find_class(module_name, Model, name) if model: if not hasattr(model, "_meta"): self.logger.error("Model %s has no _meta", name) continue if getattr(model._meta, "db_table", None) != name: self.logger.error("Table name mismatch") continue break if not model: self.logger.error("Model not found: %s", name) self.models[name] = model return model
class Command(BaseCommand): FIX_DIRS = config.get_customized_paths("fixes") def add_arguments(self, parser): subparsers = parser.add_subparsers(dest="cmd") # subparsers.add_parser("list") # apply_parser = subparsers.add_parser("apply") apply_parser.add_argument("fixes", nargs=argparse.REMAINDER, help="Apply named fixes") def handle(self, cmd, *args, **options): return getattr(self, "handle_%s" % cmd)(*args, **options) def handle_list(self, *args, **options): fixes = set() for d in self.FIX_DIRS: if not os.path.isdir(d): continue files = os.listdir(d) if "__init__.py" not in files: print( "WARNING: %s is missed. " "Create empty file " "or all fixes from %s will be ignored" % (os.path.join(d, "__init__.py"), d), file=self.stdout, ) continue for f in files: if not f.startswith("_") and f.endswith(".py"): fixes.add(f[:-3]) for f in sorted(fixes): print(f, file=self.stdout) def get_fix(self, name): for d in self.FIX_DIRS: if os.path.isfile(os.path.join(d, "%s.py" % name)): return get_handler("noc.%s.%s.fix" % (d.replace(os.sep, "."), name)) return None def handle_apply(self, fixes=None, *args, **options): if not fixes: return # Connect to mongo from noc.core.mongo.connection import connect connect() # Apply fixes for f in fixes: fix = self.get_fix(f) if not fix: self.die("Invalid fix '%s'" % f) print("Apply %s ..." % f, file=self.stdout) fix() print("... done", file=self.stdout)
def get_datastream(self, name): """ Load datastream and return DataStream instance. Returns None when no datastream found or loading error occured """ with self.lock: datastream = self.datastreams.get(name) if not datastream: logger.info("Loading datastream %s", name) if not self.is_valid_name(name): logger.error("Invalid datastream name") return None for p in config.get_customized_paths("", prefer_custom=True): path = os.path.join(p, "services", "datastream", "streams", "%s.py" % name) if not os.path.exists(path): continue if p: # Customized datastream base_name = os.path.basename(os.path.dirname(p)) module_name = "%s.services.datastream.streams.%s" % ( base_name, name) else: # Common datastream module_name = "noc.services.datastream.streams.%s" % name datastream = self.find_class(module_name, DataStream, name) if datastream: break if not datastream: logger.error("DataStream not found: %s", name) self.datastreams[name] = datastream return datastream
def load_mibs(self): for root in config.get_customized_paths("cmibs"): logger.debug("Loading compiled MIBs from '%s'", root) for path, dirnames, filenames in os.walk(root): for f in filenames: if not f.endswith(".py") or f == "__init__.py": continue # fp = os.path.join(path, f) if root != "cmibs": # Custom script base_name = os.path.basename(os.path.dirname(root)) else: # Common script base_name = "noc" # mn = "%s.%s" % (base_name, fp[:-3].replace(os.path.sep, ".")) mn = "%s.cmibs.%s" % (base_name, f[:-3]) m = __import__(mn, {}, {}, "*") if hasattr(m, "NAME") and hasattr(m, "MIB"): name = m.NAME if name in self.loaded_mibs: logger.debug("MIB is already loaded: %s, ignoring", name) continue self.loaded_mibs.add(name) logger.debug("Loading MIB: %s", name) self.mib.update(m.MIB)
def get_profile(self, name) -> Optional[Type[BaseProfile]]: """ Load profile and return BaseProfile instance. Returns None when no profile found or loading error occured """ if name == GENERIC_PROFILE: name = "Generic" with self.lock: profile = self.profiles.get(name) if not profile: self.logger.info("Loading profile %s", name) if not self.is_valid_name(name): self.logger.error("Invalid profile name") return None for p in config.get_customized_paths("", prefer_custom=True): path = os.path.join(p, "sa", "profiles", *name.split(".")) if not os.path.exists(os.path.join(path, "profile.py")): continue if p: # Custom script base_name = os.path.basename(os.path.dirname(p)) module_name = "%s.sa.profiles.%s" % (base_name, name) else: # Common script module_name = "noc.sa.profiles.%s" % name profile = self.find_class("%s.profile" % module_name, BaseProfile, name) if profile: profile.initialize() break self.profiles[name] = profile return profile
def get_backend(cls, name): """ Look for custom auth methods in custom and load it. First check if custom method with same name exists then use bundled one. :param name: param name :return: found auth method """ m = None import logging logger = logging.getLogger(__name__) for p in config.get_customized_paths(""): if p: mm = "%s.services.login.backends.%s" % (os.path.basename( os.path.dirname(p)), name) else: mm = "noc.services.login.backends.%s" % name try: m = __import__(mm, {}, {}, "*") logger.debug("Successfuly imported %s", m) except ImportError as e: logger.debug("There was an error importing %s with %s %s", e, m, mm) if m is None: return None for a in dir(m): o = getattr(m, a) if inspect.isclass(o) and issubclass( o, BaseAuthBackend) and o.__module__ == m.__name__: return o return None
def apply_snmp_rules(mcs, script): """ Initialize SNMP rules from JSON :param script: Script class :return: """ def sort_path_key(s): """ M - Main, C - Custom, G - Generic, P - profile \\|G|P -+-+- M|3|1 C|2|0 :param s: :return: """ if s.startswith(PROFILES_PATH): return 3 if "Generic" in s else 1 else: return 2 if "Generic" in s else 0 pp = script.name.rsplit(".", 1)[0] if pp == "Generic": paths = [ p for p in config.get_customized_paths( os.path.join("sa", "profiles", "Generic", "snmp_metrics") ) ] else: v, p = pp.split(".") paths = sorted( config.get_customized_paths( os.path.join("sa", "profiles", "Generic", "snmp_metrics") ) + config.get_customized_paths(os.path.join("sa", "profiles", v, p, "snmp_metrics")), key=sort_path_key, ) for path in paths: if not os.path.exists(path): continue for root, dirs, files in os.walk(path): for f in files: if f.endswith(".json"): mcs.apply_snmp_rules_from_json(script, os.path.join(root, f))
def find_classes(self): names = set() for dn in config.get_customized_paths(os.path.join(*self.base_path)): for fn in os.listdir(dn): if fn.startswith("_") or not fn.endswith(".py"): continue name = fn[:-3] if name not in self.ignored_names: names.add(name) return names
def find_profiles(self): """ Scan all available profiles """ ns = {GENERIC_PROFILE} for px in config.get_customized_paths(os.path.join("sa", "profiles"), prefer_custom=True): px = os.path.join(px, "*", "*", "__init__.py") for path in glob.glob(px): vendor, system = path.split(os.sep)[-3:-1] ns.add("%s.%s" % (vendor, system)) with self.lock: self.all_profiles = ns
def find_datastreams(self): """ Scan all available datastreams """ names = set() for dn in config.get_customized_paths( os.path.join("services", "datastream", "streams")): for file in os.listdir(dn): if file.startswith("_") or not file.endswith(".py"): continue names.add(file[:-3]) return names
def find_models(self): """ Scan all available models """ names = set() for dn in config.get_customized_paths(os.path.join("bi", "models")): for file in os.listdir(dn): if file.startswith("_") or not file.endswith(".py"): continue name = file[:-3] if name not in self.IGNORED_MODELS: names.add(file[:-3]) return names
def find_apis(self): """ Scan all available API """ names = set() for path in config.get_customized_paths(BASE_PREFIX, prefer_custom=True): if not os.path.exists(path): continue for fn in os.listdir(path): if fn.startswith("_") or not fn.endswith(".py"): continue names.add(fn[:-3]) return names
def autodiscover(self): """ Auto-load and initialize all application classes """ if self.apps: # Do not discover site twice return self.app_count = 0 prefix = os.path.join("services", "web", "apps") # Load applications installed_apps = [ x[4:] for x in settings.INSTALLED_APPS if x.startswith("noc.") ] self.menu_roots = {} for app in installed_apps: app_path = os.path.join(prefix, app) if not os.path.isdir(app_path): continue logger.debug("Loading %s applications", app) self.menu_roots[app] = self.add_module_menu("noc.%s" % app) # Initialize application for cs in config.get_customized_paths("", prefer_custom=True): if cs: basename = os.path.basename(os.path.dirname(cs)) else: basename = "noc" for f in glob.glob("%s/*/views.py" % os.path.join(cs, app_path)): d = os.path.split(f)[0] # Skip application loading if denoted by DISABLED file if os.path.isfile(os.path.join(d, "DISABLED")): continue # site.register will be called by metaclass, registering views __import__( ".".join([basename] + f[:-3].split(os.path.sep) [len(cs.split(os.path.sep)) - 1:]), {}, {}, "*", ) # Register all collected applications for app_class in self.pending_applications: self.do_register(app_class) self.pending_applications = [] # Setup router URLs self.setup_router() # Install applications logger.info("%d applications are installed", self.app_count) # Finally, order the menu self.sort_menu()
def help_command(self, cmd): for root in config.get_customized_paths("commands"): # Python, call help path = os.path.join(root, "%s.py" % cmd) if os.path.exists(path): return subprocess.call([os.environ.get("NOC_CMD", "./noc"), cmd, "--help"]) # Shell, no help path = os.path.join(root, "%s.sh" % cmd) if os.path.exists(path): self.print("Help is not available for '%s'" % cmd) return 1 # Command not found self.print("Unknown command '%s'" % cmd) return 1
def render(self): context = self.get_context() self.logger.info("Context with data: %s" % context) pm_template_path = [] for p in config.get_customized_paths("", prefer_custom=True): if p: pm_template_path += [os.path.join(p, "templates/ddash/")] else: pm_template_path += [config.path.pm_templates] j2_env = Environment(loader=FileSystemLoader(pm_template_path)) tmpl = j2_env.get_template(self.template) data = tmpl.render(context) render = demjson.decode(data) return render
def apply_snmp_rules(mcs, script): """ Initialize SNMP rules from JSON :param script: Script class :return: """ def sort_path_key(s): k1, k2 = 1, 1 if s.startswith(os.path.join("sa", "profiles")): k1 = 0 if "Generic" in s: k2 = 0 return k1, k2 pp = script.name.rsplit(".", 1)[0] if pp == "Generic": paths = [ p for p in config.get_customized_paths( os.path.join("sa", "profiles", "Generic", "snmp_metrics")) ] else: v, p = pp.split(".") paths = sorted(config.get_customized_paths( os.path.join("sa", "profiles", "Generic", "snmp_metrics")) + config.get_customized_paths( os.path.join("sa", "profiles", v, p, "snmp_metrics")), key=sort_path_key) for path in paths: if not os.path.exists(path): continue for root, dirs, files in os.walk(path): for f in files: if f.endswith(".json"): mcs.apply_snmp_rules_from_json(script, os.path.join(root, f))
def load_all(self): if self.loaded: return # Get all probes locations dirs = config.get_customized_paths(os.path.join("cm", "validators")) # Load all probes for root in dirs: for path, dirnames, filenames in os.walk(root): for f in filenames: if not f.endswith(".py") or f == "__init__.py": continue fp = os.path.join(path, f) mn = "noc.%s" % fp[:-3].replace(os.path.sep, ".") __import__(mn, {}, {}, "*") # Prevent further loading self.loaded = True
def iter_migration_names(self): """ Yield all available migration names :return: """ for app in INSTALLED_APPS: if not app.startswith("noc."): continue app = app[4:] for path in config.get_customized_paths(app, "migrations"): if not os.path.isdir(path): continue for f_name in sorted(os.listdir(path)): if f_name == "__init__.py" or not f_name.endswith(".py"): continue yield "%s.%s" % (app, f_name[:-3])
def find_scripts(self): """ Scan all available scripts """ ns = set() # Load generic scripts generics = {} # Name -> dependencies for path in glob.glob("sa/profiles/Generic/*.py"): gn = path.rsplit(os.sep)[-1][:-3] if gn in self.protected_names: continue with open(path) as f: data = f.read() # Scan for requires = [..] match = self.rx_requires.search(data) if match: generics[gn] = [ s.strip()[1:-1] for s in match.group(1).split(",") if s.strip() ] ns.add("%s.%s" % (GENERIC_PROFILE, gn)) # Load custom scripts, Load common scripts profiles = set() for gx in config.get_customized_paths(os.path.join("sa", "profiles"), prefer_custom=True): gx = os.path.join(gx, "*", "*", "*.py") for path in glob.glob(gx): vendor, system, name = path.split(os.sep)[-3:] name = name[:-3] if name in self.protected_names: continue ns.add("%s.%s.%s" % (vendor, system, name)) profiles.add("%s.%s" % (vendor, system)) # Apply generic scripts for p in profiles: for g in generics: fgn = "%s.%s" % (p, g) if fgn in ns: # Generic overriden with common script continue if any(1 for r in generics[g] if "%s.%s" % (p, r) not in ns): continue # Unsatisfied dependency # Add generic script ns.add(fgn) # with self.lock: self.all_scripts = ns
def get_script(self, name): """ Load script and return BaseScript instance. Returns None when no script found or loading error occured """ if name in self.protected_names: return None with self.lock: script = self.scripts.get(name) if not script: self.logger.info("Loading script %s", name) if not self.is_valid_name(name): self.logger.error("Invalid script name") return None try: vendor, system, sn = name.split(".") except Exception as e: self.logger.error("Error in script name \"%s\": %s", name, e) return None is_generic = False for p in config.get_customized_paths("", prefer_custom=True): if os.path.exists( os.path.join(p, "sa", "profiles", vendor, system, "%s.py" % sn)): if p: # Custom script base_name = os.path.basename(os.path.dirname(p)) else: # Common script base_name = "noc" module_name = "%s.sa.profiles.%s" % (base_name, name) break else: # Generic script module_name = "noc.sa.profiles.Generic.%s" % sn is_generic = True # Load script script = self.find_class(module_name, BaseScript, name) # Fix generic's module if script and is_generic: # Create subclass with proper name script = type("Script", (script, ), {"name": name}) script.__module__ = "noc.sa.profiles.%s" % name self.scripts[name] = script return script
def iter_app_migrations(self, app): """ Yield all migrations for application :param app: :return: """ prev_name = None for path in config.get_customized_paths(app, "migrations"): if not os.path.isdir(path): continue for f_name in sorted(os.listdir(path)): if f_name == "__init__.py" or not f_name.endswith(".py"): continue migration = self.get_migration("%s.%s" % (app, f_name[:-3])) if prev_name: migration.add_dependency(prev_name) yield migration prev_name = migration.get_name()
def find_interfaces(self): """ Scan all available scripts """ ns = set() for gx in config.get_customized_paths( os.path.join("sa", "interfaces", "*.py")): for path in glob.glob(gx): if path in ("base.py", "__init__.py"): continue with open(path) as f: data = f.read() for match in self.rx_class.finditer(data): iname = match.group("name") fname = os.path.split(path)[1] if iname.lower() == fname[:-3]: ns.add(iname) with self.lock: self.all_interfaces = ns
def get_interface(self, name): """ Load script and return BaseScript instance. Returns None when no script found or loading error occured """ with self.lock: interface = self.interfaces.get(name) if interface: return interface logger.info("Loading interface %s", name) if not self.is_valid_name(name): logger.error("Invalid interface name") return None imname = name.lower() for p in config.get_customized_paths("", prefer_custom=True): if os.path.exists( os.path.join(p, "sa", "interfaces", "%s.py" % imname)): if p: # Custom script base_name = os.path.basename( os.path.dirname(config.path.custom_path)) else: # Common script base_name = "noc" module_name = "%s.sa.interfaces.%s" % (base_name, imname) break else: logger.error("Interface not found: %s", name) self.interfaces[name] = None return None try: sm = __import__(module_name, {}, {}, "*") for n in dir(sm): o = getattr(sm, n) if (inspect.isclass(o) and issubclass(o, BaseInterface) and o.__module__ == sm.__name__): self.interfaces[name] = o return o except Exception as e: logger.error("Failed to load interface %s: %s", name, e) self.interfaces[name] = None return None
def autodiscover(self): """ Auto-load and initialize all application classes """ if self.apps: # Do not discover site twice return # Connect to mongodb import noc.lib.nosql # noqa:F401 self.installed_applications = [] prefix = "services/web/apps" # Load applications installed_apps = [x[4:] for x in settings.INSTALLED_APPS if x.startswith("noc.")] self.menu_roots = {} for app in installed_apps: app_path = os.path.join(prefix, app) if not os.path.isdir(app_path): continue logger.debug("Loading %s applications", app) self.menu_roots[app] = self.add_module_menu("noc.%s" % app) # Initialize application for cs in config.get_customized_paths("", prefer_custom=True): if cs: basename = os.path.basename(os.path.dirname(cs)) else: basename = "noc" for f in glob.glob("%s/*/views.py" % os.path.join(cs, app_path)): d = os.path.split(f)[0] # Skip application loading if denoted by DISABLED file if os.path.isfile(os.path.join(d, "DISABLED")): continue __import__(".".join([basename] + f[:-3].split(os.path.sep)[len(cs.split(os.path.sep)) - 1:]), {}, {}, "*") # Install applications for app_class in self.installed_applications: self.install_application(app_class) logger.info("%d applications are installed", len(self.installed_applications)) self.installed_applications = [] # Finally, order the menu self.sort_menu()
def get_profile(self, name): """ Load profile and return BaseProfile instance. Returns None when no profile found or loading error occured """ if name == GENERIC_PROFILE: name = "Generic" with self.lock: profile = self.profiles.get(name) if not profile: self.logger.info("Loading profile %s", name) if not self.is_valid_name(name): self.logger.error("Invalid profile name") return None for p in config.get_customized_paths("", prefer_custom=True): path = os.path.join(p, "sa", "profiles", *name.split(".")) if os.path.exists(os.path.join(path, "__init__.py")) or os.path.exists( os.path.join(path, "profile.py") ): if p: # Custom script base_name = os.path.basename(os.path.dirname(p)) module_name = "%s.sa.profiles.%s" % (base_name, name) else: # Common script module_name = "noc.sa.profiles.%s" % name for mn in ("%s.profile" % module_name, module_name): profile = self.find_class(mn, BaseProfile, name) if profile: if not profile.__module__.endswith(".profile"): warnings.warn( "%s profile: __init__.py should be moved to profile.py" % name, RemovedInNOC1904Warning, ) profile.initialize() break if profile: break self.profiles[name] = profile return profile
def get_class(self, name): with self.lock: kls = self.classes.get(name) if not kls: self.logger.info("Loading %s", name) if not self.is_valid_name(name): self.logger.error("Invalid name: %s", name) return None for p in config.get_customized_paths("", prefer_custom=True): path = self.get_path(p, name) if not os.path.exists(path): continue base_name = os.path.basename(os.path.dirname(p)) if p else "noc" module_name = self.get_module_name(base_name, name) kls = self.find_class(module_name, self.base_cls, name) if kls: break if not kls: logger.error("DataStream not found: %s", name) self.classes[name] = kls return kls
class WelcomeApplication(ExtApplication): """ main.welcome application """ title = _("Welcome") WELCOME_PATH = config.get_customized_paths( os.path.join("services", "web", "apps", "main", "welcome", "templates", "Welcome.html.j2"), prefer_custom=True, ) @view(url="^welcome/$", access=True, api=True) def api_welcome(self, request): setup = {"installation_name": config.installation_name} for p in self.WELCOME_PATH: if not os.path.exists(p): continue with open(p) as f: tpl = Template(f.read()) return self.render_response(tpl.render(setup=setup), "text/html") # @todo: Fill context return "You are not welcome!"
def get_loader(self, name): loader = self.loaders.get(name) custom_name = os.path.basename( os.path.dirname( config.get_customized_paths("", prefer_custom=True)[0])) if not loader and custom_name: logging.info("Loading %s", name) mn = "%s.etl.portmappers.%s" % (custom_name, name) try: sm = __import__(mn, {}, {}, "*") for n in dir(sm): o = getattr(sm, n) if (inspect.isclass(o) and issubclass(o, BasePortMapper) and o.__module__ == sm.__name__): loader = o break logger.error("Loader not found: %s", name) except ImportError as e: logger.error("Failed to load: %s", e) loader = None self.loaders[name] = loader return loader