def get_path(self):
paths = []
for cp in config.get_customized_paths(self.PREFIX, prefer_custom=True):
path = os.path.join(cp, self.name)
if os.path.exists(path):
paths += [path]
return paths
def get_loader(self, name):
"""
Load loader and return BaseLoader instance.
Returns None when no loader found or loading error occured
"""
with self.lock:
loader = self.loaders.get(name)
if not loader:
logger.info("Loading loader %s", name)
for p in config.get_customized_paths("", prefer_custom=True):
base = "noc.custom" if p else "noc.core"
module_name = "%s.etl.loader.%s" % (base, name)
try:
sm = __import__(module_name, {}, {}, "*")
for n in dir(sm):
o = getattr(sm, n)
if (
inspect.isclass(o)
and issubclass(o, BaseLoader)
and o.__module__ == sm.__name__
):
loader = o
break
logger.error("Loader not found: %s", name)
except Exception as e:
logger.error("Failed to load loader %s: %s", name, e)
loader = None
if loader:
break
self.loaders[name] = loader
return loader
def get_model(self, name):
with self.lock:
model = self.models.get(name)
if not model:
self.logger.info("Loading loader %s", name)
if not self.is_valid_name(name):
self.logger.error("Invalid loader name: %s", name)
return None
for p in config.get_customized_paths("", prefer_custom=True):
path = os.path.join(p, "bi", "models", "%s.py" % name)
if not os.path.exists(path):
continue
if p:
# Customized model
base_name = os.path.basename(os.path.dirname(p))
module_name = "%s.bi.models.%s" % (base_name, name)
else:
# Common model
module_name = "noc.bi.models.%s" % name
model = self.find_class(module_name, Model, name)
if model:
if not hasattr(model, "_meta"):
self.logger.error("Model %s has no _meta", name)
continue
if getattr(model._meta, "db_table", None) != name:
self.logger.error("Table name mismatch")
continue
break
if not model:
self.logger.error("Model not found: %s", name)
self.models[name] = model
return model
class Command(BaseCommand):
FIX_DIRS = config.get_customized_paths("fixes")
def add_arguments(self, parser):
subparsers = parser.add_subparsers(dest="cmd")
#
subparsers.add_parser("list")
#
apply_parser = subparsers.add_parser("apply")
apply_parser.add_argument("fixes",
nargs=argparse.REMAINDER,
help="Apply named fixes")
def handle(self, cmd, *args, **options):
return getattr(self, "handle_%s" % cmd)(*args, **options)
def handle_list(self, *args, **options):
fixes = set()
for d in self.FIX_DIRS:
if not os.path.isdir(d):
continue
files = os.listdir(d)
if "__init__.py" not in files:
print(
"WARNING: %s is missed. "
"Create empty file "
"or all fixes from %s will be ignored" %
(os.path.join(d, "__init__.py"), d),
file=self.stdout,
)
continue
for f in files:
if not f.startswith("_") and f.endswith(".py"):
fixes.add(f[:-3])
for f in sorted(fixes):
print(f, file=self.stdout)
def get_fix(self, name):
for d in self.FIX_DIRS:
if os.path.isfile(os.path.join(d, "%s.py" % name)):
return get_handler("noc.%s.%s.fix" %
(d.replace(os.sep, "."), name))
return None
def handle_apply(self, fixes=None, *args, **options):
if not fixes:
return
# Connect to mongo
from noc.core.mongo.connection import connect
connect()
# Apply fixes
for f in fixes:
fix = self.get_fix(f)
if not fix:
self.die("Invalid fix '%s'" % f)
print("Apply %s ..." % f, file=self.stdout)
fix()
print("... done", file=self.stdout)
def get_datastream(self, name):
"""
Load datastream and return DataStream instance.
Returns None when no datastream found or loading error occured
"""
with self.lock:
datastream = self.datastreams.get(name)
if not datastream:
logger.info("Loading datastream %s", name)
if not self.is_valid_name(name):
logger.error("Invalid datastream name")
return None
for p in config.get_customized_paths("", prefer_custom=True):
path = os.path.join(p, "services", "datastream", "streams",
"%s.py" % name)
if not os.path.exists(path):
continue
if p:
# Customized datastream
base_name = os.path.basename(os.path.dirname(p))
module_name = "%s.services.datastream.streams.%s" % (
base_name, name)
else:
# Common datastream
module_name = "noc.services.datastream.streams.%s" % name
datastream = self.find_class(module_name, DataStream, name)
if datastream:
break
if not datastream:
logger.error("DataStream not found: %s", name)
self.datastreams[name] = datastream
return datastream
def load_mibs(self):
for root in config.get_customized_paths("cmibs"):
logger.debug("Loading compiled MIBs from '%s'", root)
for path, dirnames, filenames in os.walk(root):
for f in filenames:
if not f.endswith(".py") or f == "__init__.py":
continue
# fp = os.path.join(path, f)
if root != "cmibs":
# Custom script
base_name = os.path.basename(os.path.dirname(root))
else:
# Common script
base_name = "noc"
# mn = "%s.%s" % (base_name, fp[:-3].replace(os.path.sep, "."))
mn = "%s.cmibs.%s" % (base_name, f[:-3])
m = __import__(mn, {}, {}, "*")
if hasattr(m, "NAME") and hasattr(m, "MIB"):
name = m.NAME
if name in self.loaded_mibs:
logger.debug("MIB is already loaded: %s, ignoring",
name)
continue
self.loaded_mibs.add(name)
logger.debug("Loading MIB: %s", name)
self.mib.update(m.MIB)
def get_profile(self, name) -> Optional[Type[BaseProfile]]:
"""
Load profile and return BaseProfile instance.
Returns None when no profile found or loading error occured
"""
if name == GENERIC_PROFILE:
name = "Generic"
with self.lock:
profile = self.profiles.get(name)
if not profile:
self.logger.info("Loading profile %s", name)
if not self.is_valid_name(name):
self.logger.error("Invalid profile name")
return None
for p in config.get_customized_paths("", prefer_custom=True):
path = os.path.join(p, "sa", "profiles", *name.split("."))
if not os.path.exists(os.path.join(path, "profile.py")):
continue
if p:
# Custom script
base_name = os.path.basename(os.path.dirname(p))
module_name = "%s.sa.profiles.%s" % (base_name, name)
else:
# Common script
module_name = "noc.sa.profiles.%s" % name
profile = self.find_class("%s.profile" % module_name,
BaseProfile, name)
if profile:
profile.initialize()
break
self.profiles[name] = profile
return profile
def get_backend(cls, name):
"""
Look for custom auth methods in custom and load it.
First check if custom method with same name exists then use bundled one.
:param name: param name
:return: found auth method
"""
m = None
import logging
logger = logging.getLogger(__name__)
for p in config.get_customized_paths(""):
if p:
mm = "%s.services.login.backends.%s" % (os.path.basename(
os.path.dirname(p)), name)
else:
mm = "noc.services.login.backends.%s" % name
try:
m = __import__(mm, {}, {}, "*")
logger.debug("Successfuly imported %s", m)
except ImportError as e:
logger.debug("There was an error importing %s with %s %s", e,
m, mm)
if m is None:
return None
for a in dir(m):
o = getattr(m, a)
if inspect.isclass(o) and issubclass(
o, BaseAuthBackend) and o.__module__ == m.__name__:
return o
return None
def apply_snmp_rules(mcs, script):
"""
Initialize SNMP rules from JSON
:param script: Script class
:return:
"""
def sort_path_key(s):
"""
M - Main, C - Custom, G - Generic, P - profile
\\|G|P
-+-+-
M|3|1
C|2|0
:param s:
:return:
"""
if s.startswith(PROFILES_PATH):
return 3 if "Generic" in s else 1
else:
return 2 if "Generic" in s else 0
pp = script.name.rsplit(".", 1)[0]
if pp == "Generic":
paths = [
p
for p in config.get_customized_paths(
os.path.join("sa", "profiles", "Generic", "snmp_metrics")
)
]
else:
v, p = pp.split(".")
paths = sorted(
config.get_customized_paths(
os.path.join("sa", "profiles", "Generic", "snmp_metrics")
)
+ config.get_customized_paths(os.path.join("sa", "profiles", v, p, "snmp_metrics")),
key=sort_path_key,
)
for path in paths:
if not os.path.exists(path):
continue
for root, dirs, files in os.walk(path):
for f in files:
if f.endswith(".json"):
mcs.apply_snmp_rules_from_json(script, os.path.join(root, f))
def find_classes(self):
names = set()
for dn in config.get_customized_paths(os.path.join(*self.base_path)):
for fn in os.listdir(dn):
if fn.startswith("_") or not fn.endswith(".py"):
continue
name = fn[:-3]
if name not in self.ignored_names:
names.add(name)
return names
def find_profiles(self):
"""
Scan all available profiles
"""
ns = {GENERIC_PROFILE}
for px in config.get_customized_paths(os.path.join("sa", "profiles"), prefer_custom=True):
px = os.path.join(px, "*", "*", "__init__.py")
for path in glob.glob(px):
vendor, system = path.split(os.sep)[-3:-1]
ns.add("%s.%s" % (vendor, system))
with self.lock:
self.all_profiles = ns
def find_datastreams(self):
"""
Scan all available datastreams
"""
names = set()
for dn in config.get_customized_paths(
os.path.join("services", "datastream", "streams")):
for file in os.listdir(dn):
if file.startswith("_") or not file.endswith(".py"):
continue
names.add(file[:-3])
return names
def find_models(self):
"""
Scan all available models
"""
names = set()
for dn in config.get_customized_paths(os.path.join("bi", "models")):
for file in os.listdir(dn):
if file.startswith("_") or not file.endswith(".py"):
continue
name = file[:-3]
if name not in self.IGNORED_MODELS:
names.add(file[:-3])
return names
def find_apis(self):
"""
Scan all available API
"""
names = set()
for path in config.get_customized_paths(BASE_PREFIX,
prefer_custom=True):
if not os.path.exists(path):
continue
for fn in os.listdir(path):
if fn.startswith("_") or not fn.endswith(".py"):
continue
names.add(fn[:-3])
return names
def autodiscover(self):
"""
Auto-load and initialize all application classes
"""
if self.apps:
# Do not discover site twice
return
self.app_count = 0
prefix = os.path.join("services", "web", "apps")
# Load applications
installed_apps = [
x[4:] for x in settings.INSTALLED_APPS if x.startswith("noc.")
]
self.menu_roots = {}
for app in installed_apps:
app_path = os.path.join(prefix, app)
if not os.path.isdir(app_path):
continue
logger.debug("Loading %s applications", app)
self.menu_roots[app] = self.add_module_menu("noc.%s" % app)
# Initialize application
for cs in config.get_customized_paths("", prefer_custom=True):
if cs:
basename = os.path.basename(os.path.dirname(cs))
else:
basename = "noc"
for f in glob.glob("%s/*/views.py" %
os.path.join(cs, app_path)):
d = os.path.split(f)[0]
# Skip application loading if denoted by DISABLED file
if os.path.isfile(os.path.join(d, "DISABLED")):
continue
# site.register will be called by metaclass, registering views
__import__(
".".join([basename] + f[:-3].split(os.path.sep)
[len(cs.split(os.path.sep)) - 1:]),
{},
{},
"*",
)
# Register all collected applications
for app_class in self.pending_applications:
self.do_register(app_class)
self.pending_applications = []
# Setup router URLs
self.setup_router()
# Install applications
logger.info("%d applications are installed", self.app_count)
# Finally, order the menu
self.sort_menu()
def help_command(self, cmd):
for root in config.get_customized_paths("commands"):
# Python, call help
path = os.path.join(root, "%s.py" % cmd)
if os.path.exists(path):
return subprocess.call([os.environ.get("NOC_CMD", "./noc"), cmd, "--help"])
# Shell, no help
path = os.path.join(root, "%s.sh" % cmd)
if os.path.exists(path):
self.print("Help is not available for '%s'" % cmd)
return 1
# Command not found
self.print("Unknown command '%s'" % cmd)
return 1
def render(self):
context = self.get_context()
self.logger.info("Context with data: %s" % context)
pm_template_path = []
for p in config.get_customized_paths("", prefer_custom=True):
if p:
pm_template_path += [os.path.join(p, "templates/ddash/")]
else:
pm_template_path += [config.path.pm_templates]
j2_env = Environment(loader=FileSystemLoader(pm_template_path))
tmpl = j2_env.get_template(self.template)
data = tmpl.render(context)
render = demjson.decode(data)
return render
def apply_snmp_rules(mcs, script):
"""
Initialize SNMP rules from JSON
:param script: Script class
:return:
"""
def sort_path_key(s):
k1, k2 = 1, 1
if s.startswith(os.path.join("sa", "profiles")):
k1 = 0
if "Generic" in s:
k2 = 0
return k1, k2
pp = script.name.rsplit(".", 1)[0]
if pp == "Generic":
paths = [
p for p in config.get_customized_paths(
os.path.join("sa", "profiles", "Generic", "snmp_metrics"))
]
else:
v, p = pp.split(".")
paths = sorted(config.get_customized_paths(
os.path.join("sa", "profiles", "Generic", "snmp_metrics")) +
config.get_customized_paths(
os.path.join("sa", "profiles", v, p,
"snmp_metrics")),
key=sort_path_key)
for path in paths:
if not os.path.exists(path):
continue
for root, dirs, files in os.walk(path):
for f in files:
if f.endswith(".json"):
mcs.apply_snmp_rules_from_json(script,
os.path.join(root, f))
def load_all(self):
if self.loaded:
return
# Get all probes locations
dirs = config.get_customized_paths(os.path.join("cm", "validators"))
# Load all probes
for root in dirs:
for path, dirnames, filenames in os.walk(root):
for f in filenames:
if not f.endswith(".py") or f == "__init__.py":
continue
fp = os.path.join(path, f)
mn = "noc.%s" % fp[:-3].replace(os.path.sep, ".")
__import__(mn, {}, {}, "*")
# Prevent further loading
self.loaded = True
def iter_migration_names(self):
"""
Yield all available migration names
:return:
"""
for app in INSTALLED_APPS:
if not app.startswith("noc."):
continue
app = app[4:]
for path in config.get_customized_paths(app, "migrations"):
if not os.path.isdir(path):
continue
for f_name in sorted(os.listdir(path)):
if f_name == "__init__.py" or not f_name.endswith(".py"):
continue
yield "%s.%s" % (app, f_name[:-3])
def find_scripts(self):
"""
Scan all available scripts
"""
ns = set()
# Load generic scripts
generics = {} # Name -> dependencies
for path in glob.glob("sa/profiles/Generic/*.py"):
gn = path.rsplit(os.sep)[-1][:-3]
if gn in self.protected_names:
continue
with open(path) as f:
data = f.read()
# Scan for requires = [..]
match = self.rx_requires.search(data)
if match:
generics[gn] = [
s.strip()[1:-1] for s in match.group(1).split(",")
if s.strip()
]
ns.add("%s.%s" % (GENERIC_PROFILE, gn))
# Load custom scripts, Load common scripts
profiles = set()
for gx in config.get_customized_paths(os.path.join("sa", "profiles"),
prefer_custom=True):
gx = os.path.join(gx, "*", "*", "*.py")
for path in glob.glob(gx):
vendor, system, name = path.split(os.sep)[-3:]
name = name[:-3]
if name in self.protected_names:
continue
ns.add("%s.%s.%s" % (vendor, system, name))
profiles.add("%s.%s" % (vendor, system))
# Apply generic scripts
for p in profiles:
for g in generics:
fgn = "%s.%s" % (p, g)
if fgn in ns:
# Generic overriden with common script
continue
if any(1 for r in generics[g] if "%s.%s" % (p, r) not in ns):
continue # Unsatisfied dependency
# Add generic script
ns.add(fgn)
#
with self.lock:
self.all_scripts = ns
def get_script(self, name):
"""
Load script and return BaseScript instance.
Returns None when no script found or loading error occured
"""
if name in self.protected_names:
return None
with self.lock:
script = self.scripts.get(name)
if not script:
self.logger.info("Loading script %s", name)
if not self.is_valid_name(name):
self.logger.error("Invalid script name")
return None
try:
vendor, system, sn = name.split(".")
except Exception as e:
self.logger.error("Error in script name \"%s\": %s", name,
e)
return None
is_generic = False
for p in config.get_customized_paths("", prefer_custom=True):
if os.path.exists(
os.path.join(p, "sa", "profiles", vendor, system,
"%s.py" % sn)):
if p:
# Custom script
base_name = os.path.basename(os.path.dirname(p))
else:
# Common script
base_name = "noc"
module_name = "%s.sa.profiles.%s" % (base_name, name)
break
else:
# Generic script
module_name = "noc.sa.profiles.Generic.%s" % sn
is_generic = True
# Load script
script = self.find_class(module_name, BaseScript, name)
# Fix generic's module
if script and is_generic:
# Create subclass with proper name
script = type("Script", (script, ), {"name": name})
script.__module__ = "noc.sa.profiles.%s" % name
self.scripts[name] = script
return script
def iter_app_migrations(self, app):
"""
Yield all migrations for application
:param app:
:return:
"""
prev_name = None
for path in config.get_customized_paths(app, "migrations"):
if not os.path.isdir(path):
continue
for f_name in sorted(os.listdir(path)):
if f_name == "__init__.py" or not f_name.endswith(".py"):
continue
migration = self.get_migration("%s.%s" % (app, f_name[:-3]))
if prev_name:
migration.add_dependency(prev_name)
yield migration
prev_name = migration.get_name()
def find_interfaces(self):
"""
Scan all available scripts
"""
ns = set()
for gx in config.get_customized_paths(
os.path.join("sa", "interfaces", "*.py")):
for path in glob.glob(gx):
if path in ("base.py", "__init__.py"):
continue
with open(path) as f:
data = f.read()
for match in self.rx_class.finditer(data):
iname = match.group("name")
fname = os.path.split(path)[1]
if iname.lower() == fname[:-3]:
ns.add(iname)
with self.lock:
self.all_interfaces = ns
def get_interface(self, name):
"""
Load script and return BaseScript instance.
Returns None when no script found or loading error occured
"""
with self.lock:
interface = self.interfaces.get(name)
if interface:
return interface
logger.info("Loading interface %s", name)
if not self.is_valid_name(name):
logger.error("Invalid interface name")
return None
imname = name.lower()
for p in config.get_customized_paths("", prefer_custom=True):
if os.path.exists(
os.path.join(p, "sa", "interfaces", "%s.py" % imname)):
if p:
# Custom script
base_name = os.path.basename(
os.path.dirname(config.path.custom_path))
else:
# Common script
base_name = "noc"
module_name = "%s.sa.interfaces.%s" % (base_name, imname)
break
else:
logger.error("Interface not found: %s", name)
self.interfaces[name] = None
return None
try:
sm = __import__(module_name, {}, {}, "*")
for n in dir(sm):
o = getattr(sm, n)
if (inspect.isclass(o) and issubclass(o, BaseInterface)
and o.__module__ == sm.__name__):
self.interfaces[name] = o
return o
except Exception as e:
logger.error("Failed to load interface %s: %s", name, e)
self.interfaces[name] = None
return None
def autodiscover(self):
"""
Auto-load and initialize all application classes
"""
if self.apps:
# Do not discover site twice
return
# Connect to mongodb
import noc.lib.nosql # noqa:F401
self.installed_applications = []
prefix = "services/web/apps"
# Load applications
installed_apps = [x[4:] for x in settings.INSTALLED_APPS if x.startswith("noc.")]
self.menu_roots = {}
for app in installed_apps:
app_path = os.path.join(prefix, app)
if not os.path.isdir(app_path):
continue
logger.debug("Loading %s applications", app)
self.menu_roots[app] = self.add_module_menu("noc.%s" % app)
# Initialize application
for cs in config.get_customized_paths("", prefer_custom=True):
if cs:
basename = os.path.basename(os.path.dirname(cs))
else:
basename = "noc"
for f in glob.glob("%s/*/views.py" % os.path.join(cs, app_path)):
d = os.path.split(f)[0]
# Skip application loading if denoted by DISABLED file
if os.path.isfile(os.path.join(d, "DISABLED")):
continue
__import__(".".join([basename] +
f[:-3].split(os.path.sep)[len(cs.split(os.path.sep)) - 1:]),
{}, {}, "*")
# Install applications
for app_class in self.installed_applications:
self.install_application(app_class)
logger.info("%d applications are installed", len(self.installed_applications))
self.installed_applications = []
# Finally, order the menu
self.sort_menu()
def get_profile(self, name):
"""
Load profile and return BaseProfile instance.
Returns None when no profile found or loading error occured
"""
if name == GENERIC_PROFILE:
name = "Generic"
with self.lock:
profile = self.profiles.get(name)
if not profile:
self.logger.info("Loading profile %s", name)
if not self.is_valid_name(name):
self.logger.error("Invalid profile name")
return None
for p in config.get_customized_paths("", prefer_custom=True):
path = os.path.join(p, "sa", "profiles", *name.split("."))
if os.path.exists(os.path.join(path, "__init__.py")) or os.path.exists(
os.path.join(path, "profile.py")
):
if p:
# Custom script
base_name = os.path.basename(os.path.dirname(p))
module_name = "%s.sa.profiles.%s" % (base_name, name)
else:
# Common script
module_name = "noc.sa.profiles.%s" % name
for mn in ("%s.profile" % module_name, module_name):
profile = self.find_class(mn, BaseProfile, name)
if profile:
if not profile.__module__.endswith(".profile"):
warnings.warn(
"%s profile: __init__.py should be moved to profile.py"
% name,
RemovedInNOC1904Warning,
)
profile.initialize()
break
if profile:
break
self.profiles[name] = profile
return profile
def get_class(self, name):
with self.lock:
kls = self.classes.get(name)
if not kls:
self.logger.info("Loading %s", name)
if not self.is_valid_name(name):
self.logger.error("Invalid name: %s", name)
return None
for p in config.get_customized_paths("", prefer_custom=True):
path = self.get_path(p, name)
if not os.path.exists(path):
continue
base_name = os.path.basename(os.path.dirname(p)) if p else "noc"
module_name = self.get_module_name(base_name, name)
kls = self.find_class(module_name, self.base_cls, name)
if kls:
break
if not kls:
logger.error("DataStream not found: %s", name)
self.classes[name] = kls
return kls
class WelcomeApplication(ExtApplication):
"""
main.welcome application
"""
title = _("Welcome")
WELCOME_PATH = config.get_customized_paths(
os.path.join("services", "web", "apps", "main", "welcome", "templates", "Welcome.html.j2"),
prefer_custom=True,
)
@view(url="^welcome/$", access=True, api=True)
def api_welcome(self, request):
setup = {"installation_name": config.installation_name}
for p in self.WELCOME_PATH:
if not os.path.exists(p):
continue
with open(p) as f:
tpl = Template(f.read())
return self.render_response(tpl.render(setup=setup), "text/html") # @todo: Fill context
return "You are not welcome!"
def get_loader(self, name):
loader = self.loaders.get(name)
custom_name = os.path.basename(
os.path.dirname(
config.get_customized_paths("", prefer_custom=True)[0]))
if not loader and custom_name:
logging.info("Loading %s", name)
mn = "%s.etl.portmappers.%s" % (custom_name, name)
try:
sm = __import__(mn, {}, {}, "*")
for n in dir(sm):
o = getattr(sm, n)
if (inspect.isclass(o) and issubclass(o, BasePortMapper)
and o.__module__ == sm.__name__):
loader = o
break
logger.error("Loader not found: %s", name)
except ImportError as e:
logger.error("Failed to load: %s", e)
loader = None
self.loaders[name] = loader
return loader
